| e:[["$","script",null,{"type":"application/ld+json","dangerouslySetInnerHTML":{"__html":"{\"@context\":\"https://schema.org\",\"@type\":\"BlogPosting\",\"headline\":\"Apache Pinot™ 0.12 - Consumer Record Lag\",\"datePublished\":\"2023-03-30T00:00:00.000Z\",\"dateModified\":\"2023-03-30T00:00:00.000Z\",\"description\":\"This post will explore a new API endpoint that lets you check how much Pinot is lagging when ingesting from Apache Kafka.\",\"image\":\"/static/images/twitter-card.png\",\"url\":\"https://pinot.apache.org/blog/2023-03-30-Apache-Pinot-0-12-Consumer-Record-Lag\",\"author\":[{\"@type\":\"Person\",\"name\":\"Mark Needham\"}]}"}}],["$","section",null,{"className":" px-5 pt-10 md:px-[13.313rem] md:py-16","children":[["$","$L10",null,{}],["$","article",null,{"className":"","children":["$","div",null,{"className":"mx-auto lg:flex","children":[["$","div",null,{"className":"lg:pr-12","children":[["$","header",null,{"className":"pt-6 md:pr-10","children":[["$","h1",null,{"className":"text-4xl font-semibold","children":"Apache Pinot™ 0.12 - Consumer Record Lag"}],["$","p",null,{"className":"pt-2 text-lg","children":["By: ","Mark Needham"]}],["$","p",null,{"className":"py-2 text-sm","children":["March 30th, 2023"," • ","5 min read"]}]]}],["$","div",null,{"className":"flex flex-col lg:flex-row","children":["$","main",null,{"className":"","children":["$","div",null,{"className":"prose max-w-[45rem] pb-8 pt-10 dark:prose-invert","children":[["$","p",null,{"children":["$","a",null,{"target":"_blank","rel":"noopener noreferrer","href":"https://youtu.be/JJEh_kBfJts","children":["$","img",null,{"alt":"Watch the video","src":"https://i3.ytimg.com/vi/JJEh_kBfJts/maxresdefault.jpg"}]}]}],["$","p",null,{"children":["The Apache Pinot community recently released version ",["$","a",null,{"target":"_blank","rel":"noopener noreferrer","href":"https://docs.pinot.apache.org/basics/releases/0.12.0","children":"0.12.0"}],", which has lots of goodies for you to play with. I’ve been exploring and writing about those features in a series of blog posts."]}],["$","p",null,{"children":"This post will explore a new API endpoint that lets you check how much Pinot is lagging when ingesting from Apache Kafka."}],["$","h2",null,{"id":"why-do-we-need-this","children":[["$","a",null,{"href":"#why-do-we-need-this","aria-hidden":"true","tabIndex":"-1","children":["$","span",null,{"className":"icon icon-link"}]}],"Why do we need this?"]}],["$","p",null,{"children":"A common question in the Pinot community is how to work out the consumption status of real-time tables."}],["$","p",null,{"children":"This was a tricky one to answer, but Pinot 0.12 sees the addition of a new API that lets us see exactly what’s going on."}],["$","h2",null,{"id":"worked-example","children":[["$","a",null,{"href":"#worked-example","aria-hidden":"true","tabIndex":"-1","children":["$","span",null,{"className":"icon icon-link"}]}],"Worked Example"]}],["$","p",null,{"children":"Let’s have a look at how it works with help from a worked example."}],["$","p",null,{"children":"First, we’re going to create a Kafka topic with 5 partitions:"}],["$","$L11",null,{"className":"language-bash","children":["$","code",null,{"className":"code-highlight language-bash","children":[["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token function","children":"docker"}]," ",["$","span",null,{"className":"token builtin class-name","children":"exec"}]," ",["$","span",null,{"className":"token parameter variable","children":"-it"}]," kafka-lag-blog kafka-topics.sh ",["$","span",null,{"className":"token punctuation","children":"\\"}],"\n"]}],["$","span",null,{"className":"code-line","children":["--bootstrap-server localhost:9092 ",["$","span",null,{"className":"token punctuation","children":"\\"}],"\n"]}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token parameter variable","children":"--partitions"}]," ",["$","span",null,{"className":"token number","children":"5"}]," ",["$","span",null,{"className":"token punctuation","children":"\\"}],"\n"]}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token parameter variable","children":"--topic"}]," events ",["$","span",null,{"className":"token punctuation","children":"\\"}],"\n"]}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token parameter variable","children":"--create"}],"\n"]}]]}]}],["$","p",null,{"children":"We’re going to populate this topic with data from a data generator, which is shown below:"}],["$","$L11",null,{"className":"language-python","children":["$","code",null,{"className":"code-highlight language-python","children":[["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token keyword","children":"import"}]," datetime",["$","span",null,{"className":"token punctuation","children":","}]," uuid",["$","span",null,{"className":"token punctuation","children":","}]," random",["$","span",null,{"className":"token punctuation","children":","}]," json",["$","span",null,{"className":"token punctuation","children":","}]," click",["$","span",null,{"className":"token punctuation","children":","}]," time\n"]}],["$","span",null,{"className":"code-line","children":"\n"}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token punctuation annotation decorator","children":["@click",["$","span",null,{"className":"token punctuation","children":"."}],"command"]}],["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":")"}],"\n"]}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token punctuation annotation decorator","children":["@click",["$","span",null,{"className":"token punctuation","children":"."}],"option"]}],["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token string","children":"'--sleep'"}],["$","span",null,{"className":"token punctuation","children":","}]," default",["$","span",null,{"className":"token operator","children":"="}],["$","span",null,{"className":"token number","children":"0.0"}],["$","span",null,{"className":"token punctuation","children":","}]," ",["$","span",null,{"className":"token builtin","children":"help"}],["$","span",null,{"className":"token operator","children":"="}],["$","span",null,{"className":"token string","children":"'Sleep between each message'"}],["$","span",null,{"className":"token punctuation","children":")"}],"\n"]}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token keyword","children":"def"}]," ",["$","span",null,{"className":"token function","children":"generate"}],["$","span",null,{"className":"token punctuation","children":"("}],"sleep",["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":":"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token keyword","children":"while"}]," ",["$","span",null,{"className":"token boolean","children":"True"}],["$","span",null,{"className":"token punctuation","children":":"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ts ",["$","span",null,{"className":"token operator","children":"="}]," datetime",["$","span",null,{"className":"token punctuation","children":"."}],"datetime",["$","span",null,{"className":"token punctuation","children":"."}],"now",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":"."}],"strftime",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token string","children":"\"%Y-%m-%dT%H:%M:%S.%fZ\""}],["$","span",null,{"className":"token punctuation","children":")"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token builtin","children":"id"}]," ",["$","span",null,{"className":"token operator","children":"="}]," ",["$","span",null,{"className":"token builtin","children":"str"}],["$","span",null,{"className":"token punctuation","children":"("}],"uuid",["$","span",null,{"className":"token punctuation","children":"."}],"uuid4",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":")"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" count ",["$","span",null,{"className":"token operator","children":"="}]," random",["$","span",null,{"className":"token punctuation","children":"."}],"randint",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token number","children":"0"}],["$","span",null,{"className":"token punctuation","children":","}]," ",["$","span",null,{"className":"token number","children":"1000"}],["$","span",null,{"className":"token punctuation","children":")"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token keyword","children":"print"}],["$","span",null,{"className":"token punctuation","children":"("}],"json",["$","span",null,{"className":"token punctuation","children":"."}],"dumps",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":"{"}],["$","span",null,{"className":"token string","children":"\"tsString\""}],["$","span",null,{"className":"token punctuation","children":":"}]," ts",["$","span",null,{"className":"token punctuation","children":","}]," ",["$","span",null,{"className":"token string","children":"\"uuid\""}],["$","span",null,{"className":"token punctuation","children":":"}]," ",["$","span",null,{"className":"token builtin","children":"id"}],["$","span",null,{"className":"token punctuation","children":","}]," ",["$","span",null,{"className":"token string","children":"\"count\""}],["$","span",null,{"className":"token punctuation","children":":"}]," count",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":")"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" time",["$","span",null,{"className":"token punctuation","children":"."}],"sleep",["$","span",null,{"className":"token punctuation","children":"("}],"sleep",["$","span",null,{"className":"token punctuation","children":")"}],"\n"]}],["$","span",null,{"className":"code-line","children":"\n"}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token keyword","children":"if"}]," __name__ ",["$","span",null,{"className":"token operator","children":"=="}]," ",["$","span",null,{"className":"token string","children":"'__main__'"}],["$","span",null,{"className":"token punctuation","children":":"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" generate",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":")"}],"\n"]}]]}]}],["$","p",null,{"children":"We can see an example of the messages generated by this script by running the following:"}],["$","$L11",null,{"className":"language-bash","children":["$","code",null,{"className":"code-highlight language-bash","children":["$","span",null,{"className":"code-line","children":["python datagen.py ",["$","span",null,{"className":"token parameter variable","children":"--sleep"}]," ",["$","span",null,{"className":"token number","children":"0.01"}]," ",["$","span",null,{"className":"token operator","children":[["$","span",null,{"className":"token file-descriptor important","children":"2"}],">"]}],"/dev/null ",["$","span",null,{"className":"token operator","children":"|"}]," ",["$","span",null,{"className":"token function","children":"head"}]," ",["$","span",null,{"className":"token parameter variable","children":"-n3"}]," ",["$","span",null,{"className":"token operator","children":"|"}]," jq ",["$","span",null,{"className":"token parameter variable","children":"-c"}],"\n"]}]}]}],["$","p",null,{"children":"You should see something like this:"}],["$","$L11",null,{"className":"language-json","children":["$","code",null,{"className":"code-highlight language-json","children":[["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token punctuation","children":"{"}],["$","span",null,{"className":"token property","children":"\"tsString\""}],["$","span",null,{"className":"token operator","children":":"}],["$","span",null,{"className":"token string","children":"\"2023-03-17T12:10:03.854680Z\""}],["$","span",null,{"className":"token punctuation","children":","}],["$","span",null,{"className":"token property","children":"\"uuid\""}],["$","span",null,{"className":"token operator","children":":"}],["$","span",null,{"className":"token string","children":"\"f3b7b5d3-b352-4cfb-a5e3-527f2c663143\""}],["$","span",null,{"className":"token punctuation","children":","}],["$","span",null,{"className":"token property","children":"\"count\""}],["$","span",null,{"className":"token operator","children":":"}],["$","span",null,{"className":"token number","children":"690"}],["$","span",null,{"className":"token punctuation","children":"}"}],"\n"]}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token punctuation","children":"{"}],["$","span",null,{"className":"token property","children":"\"tsString\""}],["$","span",null,{"className":"token operator","children":":"}],["$","span",null,{"className":"token string","children":"\"2023-03-17T12:10:03.864815Z\""}],["$","span",null,{"className":"token punctuation","children":","}],["$","span",null,{"className":"token property","children":"\"uuid\""}],["$","span",null,{"className":"token operator","children":":"}],["$","span",null,{"className":"token string","children":"\"eac57622-4b58-4456-bb38-96d1ef5a1ed5\""}],["$","span",null,{"className":"token punctuation","children":","}],["$","span",null,{"className":"token property","children":"\"count\""}],["$","span",null,{"className":"token operator","children":":"}],["$","span",null,{"className":"token number","children":"522"}],["$","span",null,{"className":"token punctuation","children":"}"}],"\n"]}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token punctuation","children":"{"}],["$","span",null,{"className":"token property","children":"\"tsString\""}],["$","span",null,{"className":"token operator","children":":"}],["$","span",null,{"className":"token string","children":"\"2023-03-17T12:10:03.875723Z\""}],["$","span",null,{"className":"token punctuation","children":","}],["$","span",null,{"className":"token property","children":"\"uuid\""}],["$","span",null,{"className":"token operator","children":":"}],["$","span",null,{"className":"token string","children":"\"65926a80-208a-408b-90d0-36cf74c8923a\""}],["$","span",null,{"className":"token punctuation","children":","}],["$","span",null,{"className":"token property","children":"\"count\""}],["$","span",null,{"className":"token operator","children":":"}],["$","span",null,{"className":"token number","children":"154"}],["$","span",null,{"className":"token punctuation","children":"}"}],"\n"]}]]}]}],["$","p",null,{"children":"So far, so good. Let’s now ingest this data into Kafka:"}],["$","$L11",null,{"className":"language-bash","children":["$","code",null,{"className":"code-highlight language-bash","children":[["$","span",null,{"className":"code-line","children":["python datagen.py ",["$","span",null,{"className":"token parameter variable","children":"--sleep"}]," ",["$","span",null,{"className":"token number","children":"0.01"}]," ",["$","span",null,{"className":"token operator","children":[["$","span",null,{"className":"token file-descriptor important","children":"2"}],">"]}],"/dev/null ",["$","span",null,{"className":"token operator","children":"|"}],"\n"]}],["$","span",null,{"className":"code-line","children":["jq ",["$","span",null,{"className":"token parameter variable","children":"-cr"}]," ",["$","span",null,{"className":"token parameter variable","children":"--arg"}]," sep ø ",["$","span",null,{"className":"token string","children":"'[.uuid, tostring] | join($sep)'"}]," ",["$","span",null,{"className":"token operator","children":"|"}],"\n"]}],["$","span",null,{"className":"code-line","children":["kcat ",["$","span",null,{"className":"token parameter variable","children":"-P"}]," ",["$","span",null,{"className":"token parameter variable","children":"-b"}]," localhost:9092 ",["$","span",null,{"className":"token parameter variable","children":"-t"}]," events ",["$","span",null,{"className":"token parameter variable","children":"-K"}],"\n"]}]]}]}],["$","p",null,{"children":"Next we’re going to create a Pinot schema and table. First, the schema config:"}],["$","$L11",null,{"className":"language-json","children":["$","code",null,{"className":"code-highlight language-json","children":[["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"schemaName\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"events\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"dimensionFieldSpecs\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"["}],["$","span",null,{"className":"token punctuation","children":"{"}]," ",["$","span",null,{"className":"token property","children":"\"name\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"uuid\""}],["$","span",null,{"className":"token punctuation","children":","}]," ",["$","span",null,{"className":"token property","children":"\"dataType\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"STRING\""}]," ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":"]"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"metricFieldSpecs\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"["}],["$","span",null,{"className":"token punctuation","children":"{"}]," ",["$","span",null,{"className":"token property","children":"\"name\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"count\""}],["$","span",null,{"className":"token punctuation","children":","}]," ",["$","span",null,{"className":"token property","children":"\"dataType\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"INT\""}]," ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":"]"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"dateTimeFieldSpecs\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"["}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"name\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"ts\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"dataType\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"TIMESTAMP\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"format\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"1:MILLISECONDS:EPOCH\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"granularity\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"1:MILLISECONDS\""}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token punctuation","children":"}"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token punctuation","children":"]"}],"\n"]}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token punctuation","children":"}"}],"\n"]}]]}]}],["$","p",null,{"children":"And now, the table config:"}],["$","$L11",null,{"className":"language-json","children":["$","code",null,{"className":"code-highlight language-json","children":[["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"tableName\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"events\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"tableType\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"REALTIME\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"segmentsConfig\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"timeColumnName\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"ts\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"schemaName\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"events\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"replication\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"1\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"replicasPerPartition\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"1\""}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"tableIndexConfig\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"loadMode\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"MMAP\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"streamConfigs\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"streamType\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"kafka\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"stream.kafka.topic.name\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"events\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"stream.kafka.broker.list\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"kafka-lag-blog:9093\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"stream.kafka.consumer.type\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"lowlevel\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"stream.kafka.consumer.prop.auto.offset.reset\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"smallest\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"stream.kafka.consumer.factory.class.name\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"org.apache.pinot.plugin.stream.kafka20.KafkaConsumerFactory\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"stream.kafka.decoder.class.name\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"org.apache.pinot.plugin.stream.kafka.KafkaJSONMessageDecoder\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"realtime.segment.flush.threshold.rows\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"10000000\""}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token punctuation","children":"}"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"ingestionConfig\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"transformConfigs\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"["}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"columnName\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"ts\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"transformFunction\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"FromDateTime(tsString, 'YYYY-MM-dd''T''HH:mm:ss.SSSSSS''Z''')\""}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token punctuation","children":"}"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token punctuation","children":"]"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"tenants\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"metadata\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],["$","span",null,{"className":"token punctuation","children":"}"}],"\n"]}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token punctuation","children":"}"}],"\n"]}]]}]}],["$","p",null,{"children":["We can create both the table and schema using the ",["$","em",null,{"children":"AddTable"}]," command:"]}],["$","$L11",null,{"className":"language-bash","children":["$","code",null,{"className":"code-highlight language-bash","children":[["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token function","children":"docker"}]," run ",["$","span",null,{"className":"token punctuation","children":"\\"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token parameter variable","children":"--network"}]," lag_blog ",["$","span",null,{"className":"token punctuation","children":"\\"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token parameter variable","children":"-v"}]," ",["$","span",null,{"className":"token constant environment","children":"$$PWD"}],"/config:/config ",["$","span",null,{"className":"token punctuation","children":"\\"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" apachepinot/pinot:0.12.0-arm64 AddTable ",["$","span",null,{"className":"token punctuation","children":"\\"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token parameter variable","children":"-schemaFile"}]," /config/schema.json ",["$","span",null,{"className":"token punctuation","children":"\\"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token parameter variable","children":"-tableConfigFile"}]," /config/table.json ",["$","span",null,{"className":"token punctuation","children":"\\"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token parameter variable","children":"-controllerHost"}]," ",["$","span",null,{"className":"token string","children":"\"pinot-controller-lag-blog\""}]," ",["$","span",null,{"className":"token punctuation","children":"\\"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token parameter variable","children":"-exec"}],"\n"]}]]}]}],["$","p",null,{"children":"Now let’s call the /consumingSegmentsInfo endpoint to see what’s going on:"}],["$","p",null,{"children":["$","code",null,{"children":"curl \"http://localhost:9000/tables/events/consumingSegmentsInfo\" 2>/dev/null | jq"}]}],["$","p",null,{"children":"The output of calling this end point is shown below:"}],["$","$L11",null,{"className":"language-json","children":["$","code",null,{"className":"code-highlight language-json","children":[["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"_segmentToConsumingInfoMap\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"events__0__0__20230317T1133Z\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"["}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"serverName\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"Server_172.29.0.4_8098\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"consumerState\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"CONSUMING\""}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"lastConsumedTimestamp\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token number","children":"1679052823350"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"partitionToOffsetMap\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"0\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"969\""}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"partitionOffsetInfo\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"currentOffsetsMap\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"0\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"969\""}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"latestUpstreamOffsetMap\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"0\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"969\""}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"recordsLagMap\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"0\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"0\""}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"availabilityLagMsMap\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token property","children":"\"0\""}],["$","span",null,{"className":"token operator","children":":"}]," ",["$","span",null,{"className":"token string","children":"\"26\""}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token punctuation","children":"}"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token punctuation","children":"}"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token punctuation","children":"}"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" ",["$","span",null,{"className":"token punctuation","children":"]"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":"…\n"}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token punctuation","children":"}"}],"\n"]}]]}]}],["$","p",null,{"children":["If we look under ",["$","em",null,{"children":"partitionOffsetInfo"}],", we can see what’s going on:"]}],["$","ul",null,{"children":[["$","li",null,{"children":"currentOffsetsMap is Pinot’s current offset"}],["$","li",null,{"children":"latestUpstreamOffsetMap is Kafka’s offset"}],["$","li",null,{"children":"recordsLagMap is the record lag"}],["$","li",null,{"children":"availabilityLagMsMap is the time lag"}]]}],["$","p",null,{"children":"This output is a bit unwieldy, so let’s create a bash function to tidy up the output into something that’s easier to consume:"}],["$","$L11",null,{"className":"language-python","children":["$","code",null,{"className":"code-highlight language-python","children":[["$","span",null,{"className":"code-line","children":["function consuming_info",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":")"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" curl ",["$","span",null,{"className":"token string","children":"\"http://localhost:9000/tables/events/consumingSegmentsInfo\""}]," ",["$","span",null,{"className":"token number","children":"2"}],["$","span",null,{"className":"token operator","children":">"}],["$","span",null,{"className":"token operator","children":"/"}],"dev",["$","span",null,{"className":"token operator","children":"/"}],"null ",["$","span",null,{"className":"token operator","children":"|"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" jq ",["$","span",null,{"className":"token operator","children":"-"}],"rc '",["$","span",null,{"className":"token punctuation","children":"["}],["$","span",null,{"className":"token punctuation","children":"."}],"_segmentToConsumingInfoMap ",["$","span",null,{"className":"token operator","children":"|"}]," keys",["$","span",null,{"className":"token punctuation","children":"["}],["$","span",null,{"className":"token punctuation","children":"]"}]," ",["$","span",null,{"className":"token keyword","children":"as"}]," $k ",["$","span",null,{"className":"token operator","children":"|"}]," ",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token punctuation","children":"["}],"$$k",["$","span",null,{"className":"token punctuation","children":"]"}]," ",["$","span",null,{"className":"token operator","children":"|"}]," ",["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token punctuation","children":"["}],["$","span",null,{"className":"token punctuation","children":"]"}]," ",["$","span",null,{"className":"token operator","children":"|"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" segment",["$","span",null,{"className":"token punctuation","children":":"}]," $k",["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" kafka",["$","span",null,{"className":"token punctuation","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":"."}],"partitionOffsetInfo",["$","span",null,{"className":"token punctuation","children":"."}],"currentOffsetsMap ",["$","span",null,{"className":"token operator","children":"|"}]," keys",["$","span",null,{"className":"token punctuation","children":"["}],["$","span",null,{"className":"token punctuation","children":"]"}]," ",["$","span",null,{"className":"token keyword","children":"as"}]," $k ",["$","span",null,{"className":"token operator","children":"|"}]," ",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token punctuation","children":"["}],"$$k",["$","span",null,{"className":"token punctuation","children":"]"}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" pinot",["$","span",null,{"className":"token punctuation","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":"."}],"partitionOffsetInfo",["$","span",null,{"className":"token punctuation","children":"."}],"latestUpstreamOffsetMap ",["$","span",null,{"className":"token operator","children":"|"}]," keys",["$","span",null,{"className":"token punctuation","children":"["}],["$","span",null,{"className":"token punctuation","children":"]"}]," ",["$","span",null,{"className":"token keyword","children":"as"}]," $k ",["$","span",null,{"className":"token operator","children":"|"}]," ",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token punctuation","children":"["}],"$$k",["$","span",null,{"className":"token punctuation","children":"]"}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" recordLag",["$","span",null,{"className":"token punctuation","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":"."}],"partitionOffsetInfo",["$","span",null,{"className":"token punctuation","children":"."}],"recordsLagMap ",["$","span",null,{"className":"token operator","children":"|"}]," keys",["$","span",null,{"className":"token punctuation","children":"["}],["$","span",null,{"className":"token punctuation","children":"]"}]," ",["$","span",null,{"className":"token keyword","children":"as"}]," $k ",["$","span",null,{"className":"token operator","children":"|"}]," ",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token punctuation","children":"["}],"$$k",["$","span",null,{"className":"token punctuation","children":"]"}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" timeLagMs",["$","span",null,{"className":"token punctuation","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":"."}],"partitionOffsetInfo",["$","span",null,{"className":"token punctuation","children":"."}],"availabilityLagMsMap ",["$","span",null,{"className":"token operator","children":"|"}]," keys",["$","span",null,{"className":"token punctuation","children":"["}],["$","span",null,{"className":"token punctuation","children":"]"}]," ",["$","span",null,{"className":"token keyword","children":"as"}]," $k ",["$","span",null,{"className":"token operator","children":"|"}]," ",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token punctuation","children":"["}],"$$k",["$","span",null,{"className":"token punctuation","children":"]"}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":")"}],"\n"]}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":"]"}]," ",["$","span",null,{"className":"token operator","children":"|"}]," ",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token punctuation","children":"["}],["$","span",null,{"className":"token number","children":"0"}],["$","span",null,{"className":"token punctuation","children":"]"}]," ",["$","span",null,{"className":"token operator","children":"|"}],"keys_unsorted ",["$","span",null,{"className":"token operator","children":"|"}]," @tsv",["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":","}]," ",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token punctuation","children":"["}],["$","span",null,{"className":"token punctuation","children":"]"}]," ",["$","span",null,{"className":"token operator","children":"|"}],["$","span",null,{"className":"token builtin","children":"map"}],["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token punctuation","children":")"}]," ",["$","span",null,{"className":"token operator","children":"|"}],"@tsv",["$","span",null,{"className":"token punctuation","children":")"}],"' ",["$","span",null,{"className":"token operator","children":"|"}]," column ",["$","span",null,{"className":"token operator","children":"-"}],"t\n"]}],["$","span",null,{"className":"code-line","children":[" printf ",["$","span",null,{"className":"token string","children":"\"\\n\""}],"\n"]}],["$","span",null,{"className":"code-line","children":"\n"}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token punctuation","children":"}"}],"\n"]}]]}]}],["$","p",null,{"children":"Let’s call the function:"}],["$","p",null,{"children":["$","code",null,{"children":"consuming\\_info"}]}],["$","p",null,{"children":"We’ll see the following output:"}],["$","p",null,{"children":["$","img",null,{"alt":"Consumer record lag output","src":"https://www.datocms-assets.com/75153/1680190272-image2.png","title":"Consumer record lag output"}]}],["$","p",null,{"children":"Now let’s put it in a script and call the watch command so that it will be refreshed every couple of seconds:"}],["$","$L11",null,{"className":"language-python","children":["$","code",null,{"className":"code-highlight language-python","children":[["$","span",null,{"className":"code-line","children":["!",["$","span",null,{"className":"token comment","children":"#/bin/bash"}],"\n"]}],["$","span",null,{"className":"code-line","children":"\n"}],["$","span",null,{"className":"code-line","children":["function consuming_info",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":")"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" curl ",["$","span",null,{"className":"token string","children":"\"http://localhost:9000/tables/events/consumingSegmentsInfo\""}]," ",["$","span",null,{"className":"token number","children":"2"}],["$","span",null,{"className":"token operator","children":">"}],["$","span",null,{"className":"token operator","children":"/"}],"dev",["$","span",null,{"className":"token operator","children":"/"}],"null ",["$","span",null,{"className":"token operator","children":"|"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" jq ",["$","span",null,{"className":"token operator","children":"-"}],"rc '",["$","span",null,{"className":"token punctuation","children":"["}],["$","span",null,{"className":"token punctuation","children":"."}],"_segmentToConsumingInfoMap ",["$","span",null,{"className":"token operator","children":"|"}]," keys",["$","span",null,{"className":"token punctuation","children":"["}],["$","span",null,{"className":"token punctuation","children":"]"}]," ",["$","span",null,{"className":"token keyword","children":"as"}]," $k ",["$","span",null,{"className":"token operator","children":"|"}]," ",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token punctuation","children":"["}],"$$k",["$","span",null,{"className":"token punctuation","children":"]"}]," ",["$","span",null,{"className":"token operator","children":"|"}]," ",["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token punctuation","children":"["}],["$","span",null,{"className":"token punctuation","children":"]"}]," ",["$","span",null,{"className":"token operator","children":"|"}]," ",["$","span",null,{"className":"token punctuation","children":"{"}],"\n"]}],["$","span",null,{"className":"code-line","children":[" segment",["$","span",null,{"className":"token punctuation","children":":"}]," $k",["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" kafka",["$","span",null,{"className":"token punctuation","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":"."}],"partitionOffsetInfo",["$","span",null,{"className":"token punctuation","children":"."}],"currentOffsetsMap ",["$","span",null,{"className":"token operator","children":"|"}]," keys",["$","span",null,{"className":"token punctuation","children":"["}],["$","span",null,{"className":"token punctuation","children":"]"}]," ",["$","span",null,{"className":"token keyword","children":"as"}]," $k ",["$","span",null,{"className":"token operator","children":"|"}]," ",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token punctuation","children":"["}],"$$k",["$","span",null,{"className":"token punctuation","children":"]"}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" pinot",["$","span",null,{"className":"token punctuation","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":"."}],"partitionOffsetInfo",["$","span",null,{"className":"token punctuation","children":"."}],"latestUpstreamOffsetMap ",["$","span",null,{"className":"token operator","children":"|"}]," keys",["$","span",null,{"className":"token punctuation","children":"["}],["$","span",null,{"className":"token punctuation","children":"]"}]," ",["$","span",null,{"className":"token keyword","children":"as"}]," $k ",["$","span",null,{"className":"token operator","children":"|"}]," ",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token punctuation","children":"["}],"$$k",["$","span",null,{"className":"token punctuation","children":"]"}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" recordLag",["$","span",null,{"className":"token punctuation","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":"."}],"partitionOffsetInfo",["$","span",null,{"className":"token punctuation","children":"."}],"recordsLagMap ",["$","span",null,{"className":"token operator","children":"|"}]," keys",["$","span",null,{"className":"token punctuation","children":"["}],["$","span",null,{"className":"token punctuation","children":"]"}]," ",["$","span",null,{"className":"token keyword","children":"as"}]," $k ",["$","span",null,{"className":"token operator","children":"|"}]," ",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token punctuation","children":"["}],"$$k",["$","span",null,{"className":"token punctuation","children":"]"}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":","}],"\n"]}],["$","span",null,{"className":"code-line","children":[" timeLagMs",["$","span",null,{"className":"token punctuation","children":":"}]," ",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":"."}],"partitionOffsetInfo",["$","span",null,{"className":"token punctuation","children":"."}],"availabilityLagMsMap ",["$","span",null,{"className":"token operator","children":"|"}]," keys",["$","span",null,{"className":"token punctuation","children":"["}],["$","span",null,{"className":"token punctuation","children":"]"}]," ",["$","span",null,{"className":"token keyword","children":"as"}]," $k ",["$","span",null,{"className":"token operator","children":"|"}]," ",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token punctuation","children":"["}],"$$k",["$","span",null,{"className":"token punctuation","children":"]"}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":")"}],"\n"]}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token punctuation","children":"}"}],["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":"]"}]," ",["$","span",null,{"className":"token operator","children":"|"}]," ",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token punctuation","children":"["}],["$","span",null,{"className":"token number","children":"0"}],["$","span",null,{"className":"token punctuation","children":"]"}]," ",["$","span",null,{"className":"token operator","children":"|"}],"keys_unsorted ",["$","span",null,{"className":"token operator","children":"|"}]," @tsv",["$","span",null,{"className":"token punctuation","children":")"}],["$","span",null,{"className":"token punctuation","children":","}]," ",["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token punctuation","children":"["}],["$","span",null,{"className":"token punctuation","children":"]"}]," ",["$","span",null,{"className":"token operator","children":"|"}],["$","span",null,{"className":"token builtin","children":"map"}],["$","span",null,{"className":"token punctuation","children":"("}],["$","span",null,{"className":"token punctuation","children":"."}],["$","span",null,{"className":"token punctuation","children":")"}]," ",["$","span",null,{"className":"token operator","children":"|"}],"@tsv",["$","span",null,{"className":"token punctuation","children":")"}],"' ",["$","span",null,{"className":"token operator","children":"|"}]," column ",["$","span",null,{"className":"token operator","children":"-"}],"t\n"]}],["$","span",null,{"className":"code-line","children":[" printf ",["$","span",null,{"className":"token string","children":"\"\\n\""}],"\n"]}],["$","span",null,{"className":"code-line","children":[["$","span",null,{"className":"token punctuation","children":"}"}],"\n"]}],["$","span",null,{"className":"code-line","children":"\n"}],["$","span",null,{"className":"code-line","children":["export ",["$","span",null,{"className":"token operator","children":"-"}],"f consuming_info\n"]}],["$","span",null,{"className":"code-line","children":["watch bash ",["$","span",null,{"className":"token operator","children":"-"}],"c consuming_info\n"]}],["$","span",null,{"className":"code-line","children":"\n"}]]}]}],["$","p",null,{"children":"Give permissions to run it as a script:"}],["$","p",null,{"children":["$","code",null,{"children":"chmod u+x watch\\_consuming\\_info.sh"}]}],["$","p",null,{"children":"And finally, run it:"}],["$","p",null,{"children":["$","code",null,{"children":"./watch\\_consuming\\_info.sh"}]}],["$","p",null,{"children":"This will print out a new table every two seconds. Let’s now make things more interesting by removing the sleep from our ingestion command:"}],["$","$L11",null,{"className":"language-bash","children":["$","code",null,{"className":"code-highlight language-bash","children":[["$","span",null,{"className":"code-line","children":["python datagen.py ",["$","span",null,{"className":"token operator","children":[["$","span",null,{"className":"token file-descriptor important","children":"2"}],">"]}],"/dev/null ",["$","span",null,{"className":"token operator","children":"|"}],"\n"]}],["$","span",null,{"className":"code-line","children":["jq ",["$","span",null,{"className":"token parameter variable","children":"-cr"}]," ",["$","span",null,{"className":"token parameter variable","children":"--arg"}]," sep ø ",["$","span",null,{"className":"token string","children":"'[.uuid, tostring] | join($sep)'"}]," ",["$","span",null,{"className":"token operator","children":"|"}],"\n"]}],["$","span",null,{"className":"code-line","children":["kcat ",["$","span",null,{"className":"token parameter variable","children":"-P"}]," ",["$","span",null,{"className":"token parameter variable","children":"-b"}]," localhost:9092 ",["$","span",null,{"className":"token parameter variable","children":"-t"}]," events -Kø\n"]}]]}]}],["$","p",null,{"children":"And now if we look at the watch output:"}],["$","p",null,{"children":["$","img",null,{"alt":"Apache Pinot Consumer Record Lag","src":"https://www.datocms-assets.com/75153/1680190286-image1.png","title":"Apache Pinot Consumer Record Lag"}]}],["$","p",null,{"children":"We get some transitory lag, but it generally goes away by the next time the command is run."}],["$","h2",null,{"id":"summary","children":[["$","a",null,{"href":"#summary","aria-hidden":"true","tabIndex":"-1","children":["$","span",null,{"className":"icon icon-link"}]}],"Summary"]}],["$","p",null,{"children":"I love this feature, and it solves a problem I’ve struggled with when using my datasets. I hope you’ll find it just as useful."}],["$","p",null,{"children":["Give it a try, and let us know how you get on. If you have any questions about this feature, feel free to join us on ",["$","a",null,{"target":"_blank","rel":"noopener noreferrer","href":"https://stree.ai/slack","children":"Slack"}],", where we’ll be happy to help you out."]}]]}]}]}]]}],["$","aside",null,{"className":"mt-10 hidden border-l-2 pl-5 lg:sticky lg:top-1 lg:block lg:h-full","children":["$","section",null,{"className":"sticky top-0 mb-4 w-[15.375rem]","children":[["$","div",null,{"className":"flex flex-col space-y-1.5 pb-3","children":["$","h3",null,{"className":"text-sm font-semibold leading-snug text-neutral-500 dark:text-neutral-100","children":"Table of Contents"}]}],["$","$L12",null,{"chapters":[{"value":"Why do we need this?","url":"#why-do-we-need-this","depth":2},{"value":"Worked Example","url":"#worked-example","depth":2},{"value":"Summary","url":"#summary","depth":2}]}]]}]}]]}]}]]}]] |