Google Git
Sign in
apache / pinot-site / 814893d8ac83b20cc413005e6f5065787252e9e0 / . / blog / tags / consumer-record-lag / index.html
blob: 99f72abcbdb46c24baeb76ce951a285e18d358f7 [file] [log] [blame]
<!doctype html>
<html lang="en" dir="ltr">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<meta name="generator" content="Docusaurus v2.0.0-beta.1">
<link rel="alternate" type="application/rss+xml" href="/blog/rss.xml" title="Apache Pinot™ Blog RSS Feed">
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Pinot™ Blog Atom Feed">
<link rel="preconnect" href="https://www.google-analytics.com">
<script>window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)},ga.l=+new Date,ga("create","UA-157446650-1","auto"),ga("send","pageview")</script>
<script async src="https://www.google-analytics.com/analytics.js"></script>
<link rel="search" type="application/opensearchdescription+xml" title="Apache Pinot™" href="/opensearch.xml">
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Ubuntu|Roboto|Source+Code+Pro">
<link rel="stylesheet" href="https://at-ui.github.io/feather-font/css/iconfont.css"><title data-react-helmet="true">One post tagged with &quot;consumer record lag&quot; | Apache Pinot™</title><meta data-react-helmet="true" property="og:title" content="One post tagged with &quot;consumer record lag&quot; | Apache Pinot™"><meta data-react-helmet="true" property="og:url" content="https://pinot.apache.org/blog/tags/consumer-record-lag"><meta data-react-helmet="true" name="docsearch:language" content="en"><meta data-react-helmet="true" name="docsearch:docusaurus_tag" content="blog_tags_posts"><meta data-react-helmet="true" name="keywords" content="Apache Pinot,Apache Pinot 0.11.0,hybrid tables,consumer record lag,Kafka"><link data-react-helmet="true" rel="shortcut icon" href="/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://pinot.apache.org/blog/tags/consumer-record-lag"><link data-react-helmet="true" rel="alternate" href="https://pinot.apache.org/blog/tags/consumer-record-lag" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://pinot.apache.org/blog/tags/consumer-record-lag" hreflang="x-default"><link data-react-helmet="true" rel="preconnect" href="https://BH4D9OD16A-dsn.algolia.net" crossorigin="anonymous"><link rel="stylesheet" href="/assets/css/styles.d071271f.css">
<link rel="preload" href="/assets/js/runtime~main.997335bd.js" as="script">
<link rel="preload" href="/assets/js/main.0ce7f94c.js" as="script">
</head>
<body>
<script>!function(){function e(e){document.documentElement.setAttribute("data-theme",e)}var t=function(){var e=null;try{e=localStorage.getItem("theme")}catch(e){}return e}();null!==t?e(t):window.matchMedia("(prefers-color-scheme: dark)").matches?e("dark"):window.matchMedia("(prefers-color-scheme: light)").matches?e("light"):e("dark")}()</script><div id="__docusaurus">
<div><a href="#main" class="skipToContent_OuoZ shadow--md">Skip to main content</a></div><nav class="navbar navbar--fixed-top navbarHideable_RReh"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle clean-btn" type="button" tabindex="0"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><img src="/img/pinot-navbar-logo-722f37.svg" alt="Pinot" class="themedImage_TMUO themedImage--light_4Vu1 navbar__logo"><img src="/img/pinot-navbar-logo-722f37.svg" alt="Pinot" class="themedImage_TMUO themedImage--dark_uzRr navbar__logo"></a></div><div class="navbar__items navbar__items--right"><a href="https://docs.pinot.apache.org/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">Docs</a><a class="navbar__item navbar__link" href="/download">Download</a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/blog">Blog</a><a href="https://github.com/apache/pinot" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link"><span>GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_wgqa"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></span></a><div class="react-toggle displayOnlyInLargeViewport_cxYs react-toggle--checked react-toggle--disabled"><div class="react-toggle-track" role="button" tabindex="-1"><div class="react-toggle-track-check"><span class="toggle_iYfV" style="margin-left:2px">🌙</span></div><div class="react-toggle-track-x"><span class="toggle_iYfV" style="margin-left:2px">☀️</span></div><div class="react-toggle-thumb"></div></div><input type="checkbox" checked="" class="react-toggle-screenreader-only" aria-label="Switch between dark and light mode"></div><div class="searchBox_NKBi"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/"><img src="/img/pinot-navbar-logo-722f37.svg" alt="Pinot" class="themedImage_TMUO themedImage--light_4Vu1 navbar__logo"><img src="/img/pinot-navbar-logo-722f37.svg" alt="Pinot" class="themedImage_TMUO themedImage--dark_uzRr navbar__logo"></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a href="https://docs.pinot.apache.org/" target="_blank" rel="noopener noreferrer" class="menu__link">Docs</a></li><li class="menu__list-item"><a class="menu__link" href="/download">Download</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/blog">Blog</a></li><li class="menu__list-item"><a href="https://github.com/apache/pinot" target="_blank" rel="noopener noreferrer" class="menu__link"><span>GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_wgqa"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></span></a></li></ul></div></div></div></nav><div class="main-wrapper blog-wrapper blog-tags-post-page"><div class="container margin-vert--lg"><div class="row"><aside class="col col--3"><nav class="sidebar_q+wC thin-scrollbar" aria-label="Blog recent posts navigation"><div class="sidebarItemTitle_9G5K margin-bottom--md">All our posts</div><ul class="sidebarItemList_6T4b"><li class="sidebarItem_cjdF"><a class="sidebarItemLink_zyXk" href="/blog/2023/09/19/Annoucing-Apache-Pinot-1-0">Announcing Apache Pinot 1.0™</a></li><li class="sidebarItem_cjdF"><a class="sidebarItemLink_zyXk" href="/blog/2023/05/11/Geospatial-Indexing-in-Apache-Pinot">Geospatial Indexing in Apache Pinot</a></li><li class="sidebarItem_cjdF"><a class="sidebarItemLink_zyXk" href="/blog/2023/03/30/Apache-Pinot-0-12-Consumer-Record-Lag">Apache Pinot™ 0.12 - Consumer Record Lag</a></li><li class="sidebarItem_cjdF"><a class="sidebarItemLink_zyXk" href="/blog/2023/02/21/Apache-Pinot-0-12-Configurable-Time-Boundary">Apache Pinot™ 0.12 - Configurable Time Boundary</a></li><li class="sidebarItem_cjdF"><a class="sidebarItemLink_zyXk" href="/blog/2023/01/29/Apache-Pinot-Deduplication-on-Real-Time-Tables">Apache Pinot™ 0.11 - Deduplication on Real-Time Tables</a></li><li class="sidebarItem_cjdF"><a class="sidebarItemLink_zyXk" href="/blog/2022/11/28/Apache-Pinot-Pausing-Real-Time-Ingestion">Apache Pinot™ 0.11 - Pausing Real-Time Ingestion</a></li><li class="sidebarItem_cjdF"><a class="sidebarItemLink_zyXk" href="/blog/2022/11/22/Apache-Pinot-Timestamp-Indexes">Apache Pinot™ 0.11 - Timestamp Indexes</a></li><li class="sidebarItem_cjdF"><a class="sidebarItemLink_zyXk" href="/blog/2022/11/17/Apache Pinot-Inserts-from-SQL">Apache Pinot™ 0.11 - Inserts from SQL</a></li><li class="sidebarItem_cjdF"><a class="sidebarItemLink_zyXk" href="/blog/2022/11/08/Apache Pinot-How-do-I-see-my-indexes">Apache Pinot™ 0.11 - How do I see my indexes?</a></li><li class="sidebarItem_cjdF"><a class="sidebarItemLink_zyXk" href="/blog/2022/08/02/GapFill-Function-For-Time-Series-Datasets-In-Pinot">GapFill Function For Time-Series Datasets In Pinot</a></li></ul></nav></aside><main class="col col--7"><header class="margin-bottom--xl"><h1>One post tagged with &quot;consumer record lag&quot;</h1><a href="/blog/tags">View All Tags</a></header><article class="margin-bottom--xl"><header><h2 class="blogPostTitle_d4p0"><a href="/blog/2023/03/30/Apache-Pinot-0-12-Consumer-Record-Lag">Apache Pinot™ 0.12 - Consumer Record Lag</a></h2><div class="blogPostData_-Im+ margin-vert--md"><time datetime="2023-03-30T00:00:00.000Z">March 30, 2023</time> · 5 min read</div><div class="avatar margin-vert--md"><a href="https://www.linkedin.com/in/markhneedham/" target="_blank" rel="noopener noreferrer" class="avatar__photo-link avatar__photo"><img src="https://www.datocms-assets.com/75153/1661544338-mark-needham.png" alt="Mark Needham"></a><div class="avatar__intro"><div class="avatar__name"><a href="https://www.linkedin.com/in/markhneedham/" target="_blank" rel="noopener noreferrer">Mark Needham</a></div><small class="avatar__subtitle">Mark Needham</small></div></div></header><div class="markdown"><p><a href="https://youtu.be/JJEh_kBfJts" target="_blank" rel="noopener noreferrer"><img src="https://i3.ytimg.com/vi/JJEh_kBfJts/maxresdefault.jpg" alt="Watch the video"></a></p><p>The Apache Pinot community recently released version <a href="https://docs.pinot.apache.org/basics/releases/0.12.0" target="_blank" rel="noopener noreferrer">0.12.0</a>, which has lots of goodies for you to play with. I’ve been exploring and writing about those features in a series of blog posts.</p><p>This post will explore a new API endpoint that lets you check how much Pinot is lagging when ingesting from Apache Kafka.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor" id="why-do-we-need-this"></a>Why do we need this?<a class="hash-link" href="#why-do-we-need-this" title="Direct link to heading">#</a></h2><p>A common question in the Pinot community is how to work out the consumption status of real-time tables. </p><p>This was a tricky one to answer, but Pinot 0.12 sees the addition of a new API that lets us see exactly what’s going on.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor" id="worked-example"></a>Worked Example<a class="hash-link" href="#worked-example" title="Direct link to heading">#</a></h2><p>Let’s have a look at how it works with help from a worked example. </p><p>First, we’re going to create a Kafka topic with 5 partitions:</p><div class="codeBlockContainer_J+bg"><div class="codeBlockContent_csEI bash"><pre tabindex="0" class="prism-code language-bash codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token function" style="color:rgb(80, 250, 123)">docker</span><span class="token plain"> </span><span class="token builtin class-name" style="color:rgb(189, 147, 249)">exec</span><span class="token plain"> -it kafka-lag-blog kafka-topics.sh </span><span class="token punctuation" style="color:rgb(248, 248, 242)">\</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">--bootstrap-server localhost:9092 </span><span class="token punctuation" style="color:rgb(248, 248, 242)">\</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">--partitions </span><span class="token number">5</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">\</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">--topic events </span><span class="token punctuation" style="color:rgb(248, 248, 242)">\</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">--create </span></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><p>We’re going to populate this topic with data from a data generator, which is shown below:</p><div class="codeBlockContainer_J+bg"><div class="codeBlockContent_csEI python"><pre tabindex="0" class="prism-code language-python codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">import</span><span class="token plain"> datetime</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"> uuid</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"> random</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"> click</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"> time</span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block">
</span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token decorator annotation punctuation" style="color:rgb(248, 248, 242)">@click</span><span class="token decorator annotation punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token decorator annotation punctuation" style="color:rgb(248, 248, 242)">command</span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token decorator annotation punctuation" style="color:rgb(248, 248, 242)">@click</span><span class="token decorator annotation punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token decorator annotation punctuation" style="color:rgb(248, 248, 242)">option</span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token string" style="color:rgb(255, 121, 198)">&#x27;--sleep&#x27;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"> default</span><span class="token operator">=</span><span class="token number">0.0</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(189, 147, 249)">help</span><span class="token operator">=</span><span class="token string" style="color:rgb(255, 121, 198)">&#x27;Sleep between each message&#x27;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(80, 250, 123)">generate</span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token plain">sleep</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">while</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> ts </span><span class="token operator">=</span><span class="token plain"> datetime</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token plain">datetime</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token plain">now</span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token plain">strftime</span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token string" style="color:rgb(255, 121, 198)">&quot;%Y-%m-%dT%H:%M:%S.%fZ&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token builtin" style="color:rgb(189, 147, 249)">id</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(189, 147, 249)">str</span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token plain">uuid</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token plain">uuid4</span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> count </span><span class="token operator">=</span><span class="token plain"> random</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token plain">randint</span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token number">0</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"> </span><span class="token number">1000</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">print</span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token plain">dumps</span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token string" style="color:rgb(255, 121, 198)">&quot;tsString&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> ts</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;uuid&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(189, 147, 249)">id</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> count</span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> time</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token plain">sleep</span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token plain">sleep</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block">
</span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">if</span><span class="token plain"> __name__ </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&#x27;__main__&#x27;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> generate</span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><p>We can see an example of the messages generated by this script by running the following:</p><div class="codeBlockContainer_J+bg"><div class="codeBlockContent_csEI bash"><pre tabindex="0" class="prism-code language-bash codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token plain">python datagen.py --sleep </span><span class="token number">0.01</span><span class="token plain"> </span><span class="token operator file-descriptor important">2</span><span class="token operator">&gt;</span><span class="token plain">/dev/null </span><span class="token operator">|</span><span class="token plain"> </span><span class="token function" style="color:rgb(80, 250, 123)">head</span><span class="token plain"> -n3 </span><span class="token operator">|</span><span class="token plain"> jq -c</span></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><p>You should see something like this:</p><div class="codeBlockContainer_J+bg"><div class="codeBlockContent_csEI json"><pre tabindex="0" class="prism-code language-json codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token property">&quot;tsString&quot;</span><span class="token operator">:</span><span class="token string" style="color:rgb(255, 121, 198)">&quot;2023-03-17T12:10:03.854680Z&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token property">&quot;uuid&quot;</span><span class="token operator">:</span><span class="token string" style="color:rgb(255, 121, 198)">&quot;f3b7b5d3-b352-4cfb-a5e3-527f2c663143&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token property">&quot;count&quot;</span><span class="token operator">:</span><span class="token number">690</span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token property">&quot;tsString&quot;</span><span class="token operator">:</span><span class="token string" style="color:rgb(255, 121, 198)">&quot;2023-03-17T12:10:03.864815Z&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token property">&quot;uuid&quot;</span><span class="token operator">:</span><span class="token string" style="color:rgb(255, 121, 198)">&quot;eac57622-4b58-4456-bb38-96d1ef5a1ed5&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token property">&quot;count&quot;</span><span class="token operator">:</span><span class="token number">522</span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token property">&quot;tsString&quot;</span><span class="token operator">:</span><span class="token string" style="color:rgb(255, 121, 198)">&quot;2023-03-17T12:10:03.875723Z&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token property">&quot;uuid&quot;</span><span class="token operator">:</span><span class="token string" style="color:rgb(255, 121, 198)">&quot;65926a80-208a-408b-90d0-36cf74c8923a&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token property">&quot;count&quot;</span><span class="token operator">:</span><span class="token number">154</span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><p>So far, so good. Let’s now ingest this data into Kafka:</p><div class="codeBlockContainer_J+bg"><div class="codeBlockContent_csEI bash"><pre tabindex="0" class="prism-code language-bash codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token plain">python datagen.py --sleep </span><span class="token number">0.01</span><span class="token plain"> </span><span class="token operator file-descriptor important">2</span><span class="token operator">&gt;</span><span class="token plain">/dev/null </span><span class="token operator">|</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">jq -cr --arg sep ø </span><span class="token string" style="color:rgb(255, 121, 198)">&#x27;[.uuid, tostring] | join($sep)&#x27;</span><span class="token plain"> </span><span class="token operator">|</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">kcat -P -b localhost:9092 -t events -K </span></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><p>Next we’re going to create a Pinot schema and table. First, the schema config:</p><div class="codeBlockContainer_J+bg"><div class="codeBlockContent_csEI json"><pre tabindex="0" class="prism-code language-json codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;schemaName&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;events&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;dimensionFieldSpecs&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token property">&quot;name&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;uuid&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"> </span><span class="token property">&quot;dataType&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;STRING&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;metricFieldSpecs&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token property">&quot;name&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"> </span><span class="token property">&quot;dataType&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;INT&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;dateTimeFieldSpecs&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;name&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;ts&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;dataType&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;TIMESTAMP&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;format&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;1:MILLISECONDS:EPOCH&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;granularity&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;1:MILLISECONDS&quot;</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><p>And now, the table config:</p><div class="codeBlockContainer_J+bg"><div class="codeBlockContent_csEI json"><pre tabindex="0" class="prism-code language-json codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;tableName&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;events&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;tableType&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;REALTIME&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;segmentsConfig&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;timeColumnName&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;ts&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;schemaName&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;events&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;replication&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;replicasPerPartition&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;1&quot;</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;tableIndexConfig&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;loadMode&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;MMAP&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;streamConfigs&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;streamType&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;kafka&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;stream.kafka.topic.name&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;events&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;stream.kafka.broker.list&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;kafka-lag-blog:9093&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;stream.kafka.consumer.type&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;lowlevel&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;stream.kafka.consumer.prop.auto.offset.reset&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;smallest&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;stream.kafka.consumer.factory.class.name&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;org.apache.pinot.plugin.stream.kafka20.KafkaConsumerFactory&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;stream.kafka.decoder.class.name&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;org.apache.pinot.plugin.stream.kafka.KafkaJSONMessageDecoder&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;realtime.segment.flush.threshold.rows&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;10000000&quot;</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;ingestionConfig&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;transformConfigs&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;columnName&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;ts&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;transformFunction&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;FromDateTime(tsString, &#x27;YYYY-MM-dd&#x27;&#x27;T&#x27;&#x27;HH:mm:ss.SSSSSS&#x27;&#x27;Z&#x27;&#x27;&#x27;)&quot;</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;tenants&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;metadata&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><p>We can create both the table and schema using the <em>AddTable</em> command:</p><div class="codeBlockContainer_J+bg"><div class="codeBlockContent_csEI bash"><pre tabindex="0" class="prism-code language-bash codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token function" style="color:rgb(80, 250, 123)">docker</span><span class="token plain"> run </span><span class="token punctuation" style="color:rgb(248, 248, 242)">\</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> --network lag_blog </span><span class="token punctuation" style="color:rgb(248, 248, 242)">\</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> -v </span><span class="token environment constant" style="color:rgb(189, 147, 249)">$PWD</span><span class="token plain">/config:/config </span><span class="token punctuation" style="color:rgb(248, 248, 242)">\</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> apachepinot/pinot:0.12.0-arm64 AddTable </span><span class="token punctuation" style="color:rgb(248, 248, 242)">\</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> -schemaFile /config/schema.json </span><span class="token punctuation" style="color:rgb(248, 248, 242)">\</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> -tableConfigFile /config/table.json </span><span class="token punctuation" style="color:rgb(248, 248, 242)">\</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> -controllerHost </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;pinot-controller-lag-blog&quot;</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">\</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> -exec</span></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><p>Now let’s call the /consumingSegmentsInfo endpoint to see what’s going on:</p><p><code>curl &quot;http://localhost:9000/tables/events/consumingSegmentsInfo&quot; 2&gt;/dev/null | jq</code></p><p>The output of calling this end point is shown below:</p><div class="codeBlockContainer_J+bg"><div class="codeBlockContent_csEI json"><pre tabindex="0" class="prism-code language-json codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;_segmentToConsumingInfoMap&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;events__0__0__20230317T1133Z&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;serverName&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;Server_172.29.0.4_8098&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;consumerState&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;CONSUMING&quot;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;lastConsumedTimestamp&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token number">1679052823350</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;partitionToOffsetMap&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;0&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;969&quot;</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;partitionOffsetInfo&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;currentOffsetsMap&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;0&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;969&quot;</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;latestUpstreamOffsetMap&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;0&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;969&quot;</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;recordsLagMap&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;0&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;0&quot;</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;availabilityLagMsMap&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token property">&quot;0&quot;</span><span class="token operator">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;26&quot;</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">…</span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><p>If we look under <em>partitionOffsetInfo</em>, we can see what’s going on:</p><ul><li>currentOffsetsMap is Pinot’s current offset</li><li>latestUpstreamOffsetMap is Kafka’s offset</li><li>recordsLagMap is the record lag</li><li>availabilityLagMsMap is the time lag</li></ul><p>This output is a bit unwieldy, so let’s create a bash function to tidy up the output into something that’s easier to consume:</p><div class="codeBlockContainer_J+bg"><div class="codeBlockContent_csEI python"><pre tabindex="0" class="prism-code language-python codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token plain">function consuming_info</span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> curl </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;http://localhost:9000/tables/events/consumingSegmentsInfo&quot;</span><span class="token plain"> </span><span class="token number">2</span><span class="token operator">&gt;</span><span class="token operator">/</span><span class="token plain">dev</span><span class="token operator">/</span><span class="token plain">null </span><span class="token operator">|</span><span class="token plain"> </span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> jq </span><span class="token operator">-</span><span class="token plain">rc &#x27;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token plain">_segmentToConsumingInfoMap </span><span class="token operator">|</span><span class="token plain"> keys</span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token plain"> </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">as</span><span class="token plain"> $k </span><span class="token operator">|</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token plain">$k</span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token plain"> </span><span class="token operator">|</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token plain"> </span><span class="token operator">|</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> segment</span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> $k</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> kafka</span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token plain">partitionOffsetInfo</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token plain">currentOffsetsMap </span><span class="token operator">|</span><span class="token plain"> keys</span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token plain"> </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">as</span><span class="token plain"> $k </span><span class="token operator">|</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token plain">$k</span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> pinot</span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token plain">partitionOffsetInfo</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token plain">latestUpstreamOffsetMap </span><span class="token operator">|</span><span class="token plain"> keys</span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token plain"> </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">as</span><span class="token plain"> $k </span><span class="token operator">|</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token plain">$k</span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> recordLag</span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token plain">partitionOffsetInfo</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token plain">recordsLagMap </span><span class="token operator">|</span><span class="token plain"> keys</span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token plain"> </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">as</span><span class="token plain"> $k </span><span class="token operator">|</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token plain">$k</span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> timeLagMs</span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token plain">partitionOffsetInfo</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token plain">availabilityLagMsMap </span><span class="token operator">|</span><span class="token plain"> keys</span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token plain"> </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">as</span><span class="token plain"> $k </span><span class="token operator">|</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token plain">$k</span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token plain"> </span><span class="token operator">|</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token number">0</span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token plain"> </span><span class="token operator">|</span><span class="token plain">keys_unsorted </span><span class="token operator">|</span><span class="token plain"> @tsv</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token plain"> </span><span class="token operator">|</span><span class="token builtin" style="color:rgb(189, 147, 249)">map</span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token plain"> </span><span class="token operator">|</span><span class="token plain">@tsv</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token plain">&#x27; </span><span class="token operator">|</span><span class="token plain"> column </span><span class="token operator">-</span><span class="token plain">t</span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> printf </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;\n&quot;</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block">
</span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><p>Let’s call the function:</p><p><code>consuming\_info</code></p><p>We’ll see the following output:</p><p><img src="https://www.datocms-assets.com/75153/1680190272-image2.png" alt="Consumer record lag output" title="Consumer record lag output"></p><p>Now let’s put it in a script and call the watch command so that it will be refreshed every couple of seconds:</p><div class="codeBlockContainer_J+bg"><div class="codeBlockContent_csEI python"><pre tabindex="0" class="prism-code language-python codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token plain">!</span><span class="token comment" style="color:rgb(98, 114, 164)">#/bin/bash</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block">
</span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">function consuming_info</span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> curl </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;http://localhost:9000/tables/events/consumingSegmentsInfo&quot;</span><span class="token plain"> </span><span class="token number">2</span><span class="token operator">&gt;</span><span class="token operator">/</span><span class="token plain">dev</span><span class="token operator">/</span><span class="token plain">null </span><span class="token operator">|</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> jq </span><span class="token operator">-</span><span class="token plain">rc &#x27;</span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token plain">_segmentToConsumingInfoMap </span><span class="token operator">|</span><span class="token plain"> keys</span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token plain"> </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">as</span><span class="token plain"> $k </span><span class="token operator">|</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token plain">$k</span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token plain"> </span><span class="token operator">|</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token plain"> </span><span class="token operator">|</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> segment</span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> $k</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> kafka</span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token plain">partitionOffsetInfo</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token plain">currentOffsetsMap </span><span class="token operator">|</span><span class="token plain"> keys</span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token plain"> </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">as</span><span class="token plain"> $k </span><span class="token operator">|</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token plain">$k</span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> pinot</span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token plain">partitionOffsetInfo</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token plain">latestUpstreamOffsetMap </span><span class="token operator">|</span><span class="token plain"> keys</span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token plain"> </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">as</span><span class="token plain"> $k </span><span class="token operator">|</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token plain">$k</span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> recordLag</span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token plain">partitionOffsetInfo</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token plain">recordsLagMap </span><span class="token operator">|</span><span class="token plain"> keys</span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token plain"> </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">as</span><span class="token plain"> $k </span><span class="token operator">|</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token plain">$k</span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> timeLagMs</span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token plain">partitionOffsetInfo</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token plain">availabilityLagMsMap </span><span class="token operator">|</span><span class="token plain"> keys</span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token plain"> </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">as</span><span class="token plain"> $k </span><span class="token operator">|</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token plain">$k</span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token plain"> </span><span class="token operator">|</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token number">0</span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token plain"> </span><span class="token operator">|</span><span class="token plain">keys_unsorted </span><span class="token operator">|</span><span class="token plain"> @tsv</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token punctuation" style="color:rgb(248, 248, 242)">[</span><span class="token punctuation" style="color:rgb(248, 248, 242)">]</span><span class="token plain"> </span><span class="token operator">|</span><span class="token builtin" style="color:rgb(189, 147, 249)">map</span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token punctuation" style="color:rgb(248, 248, 242)">.</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token plain"> </span><span class="token operator">|</span><span class="token plain">@tsv</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token plain">&#x27; </span><span class="token operator">|</span><span class="token plain"> column </span><span class="token operator">-</span><span class="token plain">t</span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> printf </span><span class="token string" style="color:rgb(255, 121, 198)">&quot;\n&quot;</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block">
</span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">export </span><span class="token operator">-</span><span class="token plain">f consuming_info</span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">watch bash </span><span class="token operator">-</span><span class="token plain">c consuming_info</span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block">
</span></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><p>Give permissions to run it as a script:</p><p><code>chmod u+x watch\_consuming\_info.sh</code></p><p>And finally, run it:</p><p><code>./watch\_consuming\_info.sh</code></p><p>This will print out a new table every two seconds. Let’s now make things more interesting by removing the sleep from our ingestion command:</p><div class="codeBlockContainer_J+bg"><div class="codeBlockContent_csEI bash"><pre tabindex="0" class="prism-code language-bash codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token plain">python datagen.py </span><span class="token operator file-descriptor important">2</span><span class="token operator">&gt;</span><span class="token plain">/dev/null </span><span class="token operator">|</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">jq -cr --arg sep ø </span><span class="token string" style="color:rgb(255, 121, 198)">&#x27;[.uuid, tostring] | join($sep)&#x27;</span><span class="token plain"> </span><span class="token operator">|</span><span class="token plain"></span></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">kcat -P -b localhost:9092 -t events -Kø</span></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><p>And now if we look at the watch output:</p><p><img src="https://www.datocms-assets.com/75153/1680190286-image1.png" alt="Apache Pinot Consumer Record Lag" title="Apache Pinot Consumer Record Lag"></p><p>We get some transitory lag, but it generally goes away by the next time the command is run. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor" id="summary"></a>Summary<a class="hash-link" href="#summary" title="Direct link to heading">#</a></h2><p>I love this feature, and it solves a problem I’ve struggled with when using my datasets. I hope you’ll find it just as useful.</p><p>Give it a try, and let us know how you get on. If you have any questions about this feature, feel free to join us on <a href="https://stree.ai/slack" target="_blank" rel="noopener noreferrer">Slack</a>, where we’ll be happy to help you out.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><a class="margin-horiz--sm" href="/blog/tags/pinot">Pinot</a><a class="margin-horiz--sm" href="/blog/tags/data">Data</a><a class="margin-horiz--sm" href="/blog/tags/analytics">Analytics</a><a class="margin-horiz--sm" href="/blog/tags/user-facing-analytics">User-Facing Analytics</a><a class="margin-horiz--sm" href="/blog/tags/consumer-record-lag">consumer record lag</a><a class="margin-horiz--sm" href="/blog/tags/kafka">kafka</a></div><div class="col text--right"><a aria-label="Read more about Apache Pinot™ 0.12 - Consumer Record Lag" href="/blog/2023/03/30/Apache-Pinot-0-12-Consumer-Record-Lag"><b>Read More</b></a></div></footer></article></main></div></div></div><footer class="footer"><div class="container"><div class="row footer__links"><div class="col footer__col"><h4 class="footer__title">About</h4><ul class="footer__items"><li class="footer__item"><a href="https://docs.pinot.apache.org/" target="_blank" rel="noopener noreferrer" class="footer__link-item">What is Apache Pinot?</a></li><li class="footer__item"><a class="footer__link-item" href="/who_uses">Who uses Apache Pinot?</a></li><li class="footer__item"><a href="https://docs.pinot.apache.org/pinot-components" target="_blank" rel="noopener noreferrer" class="footer__link-item">Components</a></li><li class="footer__item"><a href="https://docs.pinot.apache.org/basics/architecture" target="_blank" rel="noopener noreferrer" class="footer__link-item">Architecture</a></li><li class="footer__item"><a href="https://docs.pinot.apache.org/plugins/plugin-architecture" target="_blank" rel="noopener noreferrer" class="footer__link-item">Plugins Architecture</a></li></ul></div><div class="col footer__col"><h4 class="footer__title">Integrations</h4><ul class="footer__items"><li class="footer__item"><a href="https://docs.pinot.apache.org/integrations/trino" target="_blank" rel="noopener noreferrer" class="footer__link-item">Trino</a></li><li class="footer__item"><a href="https://docs.pinot.apache.org/integrations/presto" target="_blank" rel="noopener noreferrer" class="footer__link-item">Presto</a></li><li class="footer__item"><a href="https://docs.pinot.apache.org/integrations/superset" target="_blank" rel="noopener noreferrer" class="footer__link-item">Superset</a></li><li class="footer__item"><a href="https://docs.pinot.apache.org/integrations/thirdeye" target="_blank" rel="noopener noreferrer" class="footer__link-item">ThirdEye</a></li></ul></div><div class="col footer__col"><h4 class="footer__title">Docs</h4><ul class="footer__items"><li class="footer__item"><a href="https://docs.pinot.apache.org/getting-started" target="_blank" rel="noopener noreferrer" class="footer__link-item">Getting Started</a></li><li class="footer__item"><a href="https://docs.pinot.apache.org/pinot-components" target="_blank" rel="noopener noreferrer" class="footer__link-item">Pinot Components</a></li><li class="footer__item"><a href="https://docs.pinot.apache.org/users" target="_blank" rel="noopener noreferrer" class="footer__link-item">User Guide</a></li><li class="footer__item"><a href="https://docs.pinot.apache.org/operators/operating-pinot" target="_blank" rel="noopener noreferrer" class="footer__link-item">Administration</a></li></ul></div><div class="col footer__col"><h4 class="footer__title">Community</h4><ul class="footer__items"><li class="footer__item"><a href="https://join.slack.com/t/apache-pinot/shared_invite/zt-5z7pav2f-yYtjZdVA~EDmrGkho87Vzw" target="_blank" rel="noopener noreferrer" class="footer__link-item">Slack</a></li><li class="footer__item"><a href="https://github.com/apache/pinot" target="_blank" rel="noopener noreferrer" class="footer__link-item">Github</a></li><li class="footer__item"><a href="https://twitter.com/ApachePinot" target="_blank" rel="noopener noreferrer" class="footer__link-item">Twitter</a></li><li class="footer__item"><a href="mailto:dev-subscribe@pinot.apache.org?Subject=SubscribeToPinot" target="_blank" rel="noopener noreferrer" class="footer__link-item">Mailing List</a></li></ul></div><div class="col footer__col"><h4 class="footer__title">Apache</h4><ul class="footer__items"><li class="footer__item"><a href="https://www.apache.org/events/current-event" target="_blank" rel="noopener noreferrer" class="footer__link-item">Events</a></li><li class="footer__item"><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Thanks</a></li><li class="footer__item"><a href="https://www.apache.org/licenses" target="_blank" rel="noopener noreferrer" class="footer__link-item">License</a></li><li class="footer__item"><a href="https://www.apache.org/security" target="_blank" rel="noopener noreferrer" class="footer__link-item">Security</a></li><li class="footer__item"><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Sponsorship</a></li><li class="footer__item"><a href="https://www.apache.org" target="_blank" rel="noopener noreferrer" class="footer__link-item">Foundation</a></li></ul></div></div><div class="footer__bottom text--center"><div class="margin-bottom--sm"><a href="https://pinot.apache.org/" target="_blank" rel="noopener noreferrer" class="footerLogoLink_94kH"><img src="/img/logo.svg" alt="Apache Pinot™" class="themedImage_TMUO themedImage--light_4Vu1 footer__logo"><img src="/img/logo.svg" alt="Apache Pinot™" class="themedImage_TMUO themedImage--dark_uzRr footer__logo"></a></div><div class="footerCopyright_-piB">Copyright © 2023 The Apache Software Foundation.<br>Apache Pinot, Pinot, Apache, the Apache feather logo, and the Apache Pinot project logo are registered trademarks of The Apache Software Foundation.<br><br>This page has references to third party software - Presto, PrestoDB, ThirdEye, Trino, TrinoDB, that are not part of the Apache Software Foundation and are not covered under the Apache License.</div></div></div></footer></div>
<script src="/assets/js/runtime~main.997335bd.js"></script>
<script src="/assets/js/main.0ce7f94c.js"></script>
</body>
</html>