blob: 9c4b0ce374586733d56326f7e1b7950fdc20569d [file] [log] [blame]
<!DOCTYPE html><html lang="en-us" class="__variable_1fc36d scroll-smooth"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" href="/_next/static/media/6905431624c34d00-s.p.woff2" as="font" crossorigin="" type="font/woff2"/><link rel="preload" as="image" href="https://www.datocms-assets.com/75153/1668701275-image4.png"/><link rel="preload" as="image" href="https://www.datocms-assets.com/75153/1668701353-image1.png"/><link rel="preload" as="image" href="https://www.datocms-assets.com/75153/1668701512-image3.png"/><link rel="preload" as="image" href="https://www.datocms-assets.com/75153/1668701654-image5.png"/><link rel="preload" as="image" href="https://www.datocms-assets.com/75153/1668701727-image2.png"/><link rel="stylesheet" href="/_next/static/css/9e925a33b1acdac1.css" crossorigin="" data-precedence="next"/><link rel="stylesheet" href="/_next/static/css/c130d1629644f070.css" crossorigin="" data-precedence="next"/><link rel="preload" as="script" fetchPriority="low" href="/_next/static/chunks/webpack-dde39ac7c1b4eb4b.js" crossorigin=""/><script src="/_next/static/chunks/fd9d1056-f172993f20f9bb67.js" async="" crossorigin=""></script><script src="/_next/static/chunks/472-928e738895d89765.js" async="" crossorigin=""></script><script src="/_next/static/chunks/main-app-344a87763a0866a6.js" async="" crossorigin=""></script><script src="/_next/static/chunks/326-3a90a6443b9c824c.js" async=""></script><script src="/_next/static/chunks/980-6e243f9cd384c7d2.js" async=""></script><script src="/_next/static/chunks/702-a2bf9fe707814b79.js" async=""></script><script src="/_next/static/chunks/app/layout-776a485845c720ef.js" async=""></script><script src="/_next/static/chunks/413-f9f40b83f7bb3f22.js" async=""></script><script src="/_next/static/chunks/app/blog/%5B...slug%5D/page-502e08b6677b55da.js" async=""></script><link rel="preload" href="https://www.googletagmanager.com/gtag/js?id=G-ZXG79NJEBY" as="script"/><title>Apache Pinot™ 0.11 - Inserts from SQL | Apache Pinot™</title><meta name="description" content="Explore the INSERT INTO clause, which makes ingesting batch data into Pinot as easy as writing a SQL query."/><meta name="robots" content="index, follow"/><meta name="googlebot" content="index, follow, max-video-preview:-1, max-image-preview:large, max-snippet:-1"/><link rel="canonical" href="https://pinot.apache.org/blog/2022/11/17/Apache%20Pinot-Inserts-from-SQL"/><link rel="alternate" type="application/rss+xml" href="https://pinot.apache.org/feed.xml"/><meta property="og:title" content="Apache Pinot™ 0.11 - Inserts from SQL"/><meta property="og:description" content="Explore the INSERT INTO clause, which makes ingesting batch data into Pinot as easy as writing a SQL query."/><meta property="og:url" content="https://pinot.apache.org/blog/2022/11/17/Apache%20Pinot-Inserts-from-SQL"/><meta property="og:site_name" content="Apache Pinot™"/><meta property="og:locale" content="en_US"/><meta property="og:image" content="https://pinot.apache.org/static/images/twitter-card.png"/><meta property="og:type" content="article"/><meta property="article:published_time" content="2022-11-17T00:00:00.000Z"/><meta property="article:modified_time" content="2022-11-17T00:00:00.000Z"/><meta property="article:author" content="Mark Needham"/><meta name="twitter:card" content="summary_large_image"/><meta name="twitter:title" content="Apache Pinot™ 0.11 - Inserts from SQL"/><meta name="twitter:description" content="Explore the INSERT INTO clause, which makes ingesting batch data into Pinot as easy as writing a SQL query."/><meta name="twitter:image" content="https://pinot.apache.org/static/images/twitter-card.png"/><meta name="next-size-adjust"/><meta http-equiv="Content-Security-Policy" content="default-src &#x27;self&#x27;;script-src &#x27;self&#x27; &#x27;unsafe-eval&#x27; &#x27;unsafe-inline&#x27; giscus.app analytics.umami.is www.youtube.com www.googletagmanager.com www.google-analytics.com;style-src &#x27;self&#x27; &#x27;unsafe-inline&#x27;;img-src * blob: data:;media-src *.s3.amazonaws.com;connect-src *;font-src &#x27;self&#x27;;frame-src www.youtube.com youtube.com giscus.app youtu.be https://www.youtube.com https://youtube.com;"/><link rel="apple-touch-icon" sizes="76x76" href="/static/favicons/apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="/static/favicons/favicon-32x32.png"/><link rel="icon" type="image/png" sizes="16x16" href="/static/favicons/favicon-16x16.png"/><link rel="manifest" href="/static/favicons/site.webmanifest"/><link rel="mask-icon" href="/static/favicons/safari-pinned-tab.svg" color="#5bbad5"/><meta name="msapplication-TileColor" content="#000000"/><meta name="theme-color" media="(prefers-color-scheme: light)" content="#fff"/><meta name="theme-color" media="(prefers-color-scheme: dark)" content="#000"/><link rel="alternate" type="application/rss+xml" href="/feed.xml"/><script src="/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js" crossorigin="" noModule=""></script></head><body class="bg-white text-black antialiased dark:bg-gray-950 dark:text-white"><script>!function(){try{var d=document.documentElement,c=d.classList;c.remove('light','dark');var e=localStorage.getItem('theme');if('system'===e||(!e&&false)){var t='(prefers-color-scheme: dark)',m=window.matchMedia(t);if(m.media!==t||m.matches){d.style.colorScheme = 'dark';c.add('dark')}else{d.style.colorScheme = 'light';c.add('light')}}else if(e){c.add(e|| '')}else{c.add('light')}if(e==='light'||e==='dark'||!e)d.style.colorScheme=e||'light'}catch(e){}}()</script><div class="mx-auto flex max-w-screen-customDesktop flex-col justify-between font-sans"><div class="inset-x-0 top-0 z-50 flex text-center text-base sm:text-left"><div class="flex w-full flex-col items-center justify-center bg-sky-200 pt-1 md:flex-row md:pt-0"><div class="flex flex-wrap items-center justify-center md:justify-start">🎉🎉🎉 Announcing the release of Apache Pinot 1.1.0</div><div class="flex items-center justify-center"><a href="https://github.com/apache/pinot/releases/tag/release-1.1.0" target="_blank" class="inline-flex items-center justify-center whitespace-nowrap rounded-md ring-offset-background transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 underline-offset-4 hover:text-vine-120 h-10 px-4 py-2 mr-2 text-base font-semibold leading-tight text-vine-100">learn more<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-arrow-right mr-2 h-5 w-5"><path d="M5 12h14"></path><path d="m12 5 7 7-7 7"></path></svg></a></div></div></div><header class="border-b-1 flex items-center justify-between border-b px-5 py-3 md:px-[4rem] md:py-4"><div class="flex"><a aria-label="" href="/"><div class="flex items-center justify-between"><div class=""><svg xmlns="http://www.w3.org/2000/svg" width="120" height="48" fill="none"><g fill="#C7154A" clip-path="url(#logo_svg__a)"><path d="M42.99 18.448c1.032-.553 2.21-.831 3.535-.831 1.542 0 2.938.38 4.187 1.14 1.248.76 2.236 1.841 2.965 3.241.728 1.402 1.091 3.025 1.091 4.872s-.363 3.482-1.091 4.903c-.729 1.424-1.717 2.525-2.965 3.307-1.25.782-2.645 1.173-4.187 1.173-1.325 0-2.493-.271-3.503-.815-1.01-.543-1.83-1.226-2.46-2.053v14.612H36V17.912h4.562v2.606c.586-.825 1.395-1.515 2.426-2.068l.002-.002m6.452 5.605c-.445-.793-1.032-1.395-1.76-1.808a4.72 4.72 0 0 0-2.362-.618c-.847 0-1.602.211-2.33.635-.728.423-1.315 1.038-1.76 1.841-.445.804-.668 1.749-.668 2.835 0 1.087.221 2.032.668 2.835.445.804 1.032 1.417 1.76 1.842a4.557 4.557 0 0 0 2.33.635 4.57 4.57 0 0 0 2.362-.652c.728-.435 1.313-1.053 1.76-1.856.445-.804.668-1.76.668-2.867s-.223-2.025-.668-2.818v-.004M62.947 17.912v18.051h-4.562V17.912h4.562m.551-6.079a2.833 2.833 0 1 1-5.666 0 2.833 2.833 0 0 1 5.666 0M82.954 19.687c1.325 1.358 1.988 3.253 1.988 5.685v10.59H80.38v-9.97c0-1.434-.358-2.537-1.075-3.307-.717-.772-1.695-1.157-2.933-1.157-1.239 0-2.254.387-2.982 1.157-.728.772-1.091 1.873-1.091 3.307v9.97h-4.562V17.91h4.562v2.248a6.322 6.322 0 0 1 2.33-1.841c.944-.445 1.981-.669 3.111-.669 2.15 0 3.889.68 5.214 2.037v.002M92.892 35.098c-1.39-.77-2.482-1.861-3.275-3.275-.794-1.411-1.19-3.041-1.19-4.888s.406-3.475 1.221-4.888a8.502 8.502 0 0 1 3.34-3.275c1.412-.772 2.987-1.157 4.725-1.157 1.739 0 3.312.387 4.725 1.157a8.5 8.5 0 0 1 3.34 3.275c.815 1.411 1.222 3.041 1.222 4.888s-.418 3.475-1.255 4.888a8.708 8.708 0 0 1-3.388 3.275c-1.424.772-3.014 1.157-4.774 1.157-1.76 0-3.301-.385-4.691-1.157m7.021-3.421c.729-.402 1.309-1.005 1.744-1.809.435-.803.651-1.781.651-2.933 0-1.715-.451-3.035-1.351-3.958-.902-.924-2.004-1.385-3.307-1.385s-2.395.461-3.275 1.385c-.88.923-1.32 2.243-1.32 3.958 0 1.715.428 3.035 1.287 3.958.858.924 1.938 1.385 3.241 1.385.825 0 1.602-.2 2.33-.603v.002M115.96 21.658v8.734c0 .608.147 1.048.44 1.32.293.271.787.406 1.482.406H120v3.845h-2.867c-3.845 0-5.766-1.868-5.766-5.605v-8.7h-2.15v-3.746h2.15V13l4.595-1v5.912h4.04v3.746h-4.042M20.03 46.757l-5.538-1.385A1.97 1.97 0 0 1 13 43.46v-5.462c0-.841.349-1.601.907-2.146a12.212 12.212 0 0 0 6.975-3.644c2.602-2.731 3.627-6.578 2.882-10.251L21 9h-4V4a1 1 0 0 0-2 0v7a1 1 0 0 1-2 0v-1a1 1 0 0 0-2 0v6.758a4.489 4.489 0 0 1 2.694-.755c2.278.095 4.156 1.934 4.297 4.21a4.501 4.501 0 0 1-6.992 4.029V29a1 1 0 0 1-2 0V7a1 1 0 0 0-2 0v2h-4L.237 21.957c-.745 3.675.279 7.52 2.882 10.251a12.202 12.202 0 0 0 6.975 3.644c.558.545.907 1.305.907 2.146V43.4c0 .938-.639 1.757-1.55 1.985l-5.48 1.37c-.57.143-.97.655-.97 1.243h18c0-.588-.4-1.1-.97-1.243v.002"></path><path d="M13.5 23a2.5 2.5 0 1 0 0-5 2.5 2.5 0 0 0 0 5M8 5a1 1 0 1 0 0-2 1 1 0 0 0 0 2M12 8a1 1 0 1 0 0-2 1 1 0 0 0 0 2M16 2a1 1 0 1 0 0-2 1 1 0 0 0 0 2"></path></g><defs><clipPath id="logo_svg__a"><path fill="#fff" d="M0 0h120v48H0z"></path></clipPath></defs></svg></div><div class="hidden h-6 text-2xl font-semibold sm:block"></div></div></a><div class="ml-[4.5rem] flex items-center gap-12 text-lg leading-5"><a target="_blank" rel="noopener noreferrer" href="https://docs.pinot.apache.org" class="hidden sm:block font-medium text-gray-900 dark:text-gray-100">Docs</a><a class="hidden sm:block font-medium text-gray-900 dark:text-gray-100" href="/download">Download</a><a class="hidden sm:block font-medium text-gray-900 dark:text-gray-100" href="/powered-by">Powered by</a><a class="hidden sm:block font-medium text-gray-900 dark:text-gray-100" href="/blog">Blog</a></div></div><button aria-label="Toggle Menu" class="sm:hidden"><svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" fill="none"><mask id="menu_svg__a" width="32" height="32" x="0" y="0" maskUnits="userSpaceOnUse" style="mask-type:alpha"><path fill="#D9D9D9" d="M0 0h32v32H0z"></path></mask><g mask="url(#menu_svg__a)"><path fill="#201F1F" d="M4.667 23.513v-2h22.666v2H4.667Zm0-6.513v-2h22.666v2H4.667Zm0-6.513v-2h22.666v2H4.667Z"></path></g></svg></button><div class="hidden gap-3 sm:flex"><button aria-label="Search"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="h-6 w-6 text-gray-900 dark:text-gray-100"><path stroke-linecap="round" stroke-linejoin="round" d="M21 21l-5.197-5.197m0 0A7.5 7.5 0 105.196 5.196a7.5 7.5 0 0010.607 10.607z"></path></svg></button><a target="_blank" rel="noopener noreferrer" href="https://github.com/apache/pinot" class="inline-flex items-center justify-center whitespace-nowrap font-medium ring-offset-background transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 border border-input bg-background hover:bg-accent hover:text-accent-foreground rounded-md px-3 py-2 text-base"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="currentColor" class="mr-2"><g clip-path="url(#github_svg__a)"><path fill-rule="evenodd" d="M12.01 0C5.369 0 0 5.5 0 12.304c0 5.44 3.44 10.043 8.212 11.673.597.122.815-.265.815-.59 0-.286-.02-1.264-.02-2.283-3.34.734-4.036-1.466-4.036-1.466-.537-1.426-1.332-1.793-1.332-1.793-1.094-.754.08-.754.08-.754 1.212.082 1.849 1.263 1.849 1.263 1.073 1.874 2.803 1.345 3.5 1.019.098-.795.417-1.345.755-1.65-2.665-.285-5.468-1.345-5.468-6.07 0-1.345.477-2.445 1.232-3.3-.119-.306-.537-1.57.12-3.26 0 0 1.014-.326 3.3 1.263.98-.27 1.989-.407 3.003-.408 1.014 0 2.048.143 3.002.408 2.287-1.59 3.301-1.263 3.301-1.263.657 1.69.239 2.954.12 3.26.775.855 1.232 1.955 1.232 3.3 0 4.725-2.803 5.764-5.488 6.07.438.387.815 1.12.815 2.281 0 1.65-.02 2.975-.02 3.382 0 .326.22.713.816.59C20.56 22.347 24 17.744 24 12.305 24.02 5.5 18.63 0 12.01 0" clip-rule="evenodd"></path></g><defs><clipPath id="github_svg__a"><path fill="#fff" d="M0 0h24v24H0z"></path></clipPath></defs></svg>3.5k</a><button class="inline-flex items-center justify-center whitespace-nowrap font-medium ring-offset-background transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 text-primary-foreground hover:bg-vine-120 rounded-md bg-vine-100 px-6 py-2 text-base"><a target="_blank" rel="noopener noreferrer" href="https://docs.pinot.apache.org/basics/getting-started">Get Started</a></button></div></header><main><script type="application/ld+json">{"@context":"https://schema.org","@type":"BlogPosting","headline":"Apache Pinot™ 0.11 - Inserts from SQL","datePublished":"2022-11-17T00:00:00.000Z","dateModified":"2022-11-17T00:00:00.000Z","description":"Explore the INSERT INTO clause, which makes ingesting batch data into Pinot as easy as writing a SQL query.","image":"/static/images/twitter-card.png","url":"https://pinot.apache.org/blog/2022-11-17-Apache Pinot-Inserts-from-SQL","author":[{"@type":"Person","name":"Mark Needham"}]}</script><section class=" px-5 pt-10 md:px-[13.313rem] md:py-16"><div class="fixed bottom-8 right-8 hidden flex-col gap-3 md:hidden"><button aria-label="Scroll To Top" class="rounded-full bg-gray-200 p-2 text-gray-500 transition-all hover:bg-gray-300 dark:bg-gray-700 dark:text-gray-400 dark:hover:bg-gray-600"><svg class="h-5 w-5" viewBox="0 0 20 20" fill="currentColor"><path fill-rule="evenodd" d="M3.293 9.707a1 1 0 010-1.414l6-6a1 1 0 011.414 0l6 6a1 1 0 01-1.414 1.414L11 5.414V17a1 1 0 11-2 0V5.414L4.707 9.707a1 1 0 01-1.414 0z" clip-rule="evenodd"></path></svg></button></div><article class=""><div class="mx-auto lg:flex"><div class="lg:pr-12"><header class="pt-6 md:pr-10"><h1 class="text-4xl font-semibold">Apache Pinot™ 0.11 - Inserts from SQL</h1><p class="pt-2 text-lg">By: <!-- -->Mark Needham</p><p class="py-2 text-sm">November 17th, 2022<!-- --> • <!-- -->4 min read</p></header><div class="flex flex-col lg:flex-row"><main class=""><div class="prose max-w-[45rem] pb-8 pt-10 dark:prose-invert"><p>The Apache Pinot community recently released version <a target="_blank" rel="noopener noreferrer" href="https://medium.com/apache-pinot-developer-blog/apache-pinot-0-11-released-d564684df5d4">0.11.0</a>, which has lots of goodies for you to play with. This is the second in a series of blog posts showing off some of the new features in this release.</p><p>In this post, we’re going to explore the <a target="_blank" rel="noopener noreferrer" href="https://docs.pinot.apache.org/basics/data-import/from-query-console">INSERT INTO clause</a>, which makes ingesting batch data into Pinot as easy as writing a SQL query.</p><h2 id="batch-importing-the-job-specification"><a href="#batch-importing-the-job-specification" aria-hidden="true" tabindex="-1"><span class="icon icon-link"></span></a>Batch importing: The Job Specification</h2><p>The power of this new clause is only fully appreciated if we look at what we had to do before it existed.</p><p>In the <a target="_blank" rel="noopener noreferrer" href="https://www.youtube.com/watch?v=1EMBx1XeI9o">Batch Import JSON from Amazon S3 into Apache Pinot | StarTree Recipes</a> video (and <a target="_blank" rel="noopener noreferrer" href="https://dev.startree.ai/docs/pinot/recipes/ingest-csv-files-from-s3">accompanying developer guide</a>), we showed how to ingest data into Pinot from an S3 bucket.</p><p>The contents of that bucket are shown in the screenshot below:</p><p><img alt="Sample data ingested into Apache Pinot from a S3 bucket" src="https://www.datocms-assets.com/75153/1668701275-image4.png" title="Sample data ingested into Apache Pinot from a S3 bucket"/></p><p>Let’s quickly recap the steps that we had to do to import those files into Pinot. We have a table called events, which has the following schema:</p><p><img alt="Events schema table" src="https://www.datocms-assets.com/75153/1668701353-image1.png" title="Events schema table"/></p><p>We first created a job specification file, which contains a description of our import job. The job file is shown below:</p><div class="relative"><pre><code class="code-highlight language-yaml"><span class="code-line"><span class="token atrule key">executionFrameworkSpec</span><span class="token punctuation">:</span>
</span><span class="code-line"> <span class="token atrule key">name</span><span class="token punctuation">:</span> <span class="token string">&#x27;standalone&#x27;</span>
</span><span class="code-line"> <span class="token atrule key">segmentGenerationJobRunnerClassName</span><span class="token punctuation">:</span> <span class="token string">&#x27;org.apache.pinot.plugin.ingestion.batch.standalone.SegmentGenerationJobRunner&#x27;</span>
</span><span class="code-line"> <span class="token atrule key">segmentTarPushJobRunnerClassName</span><span class="token punctuation">:</span> <span class="token string">&#x27;org.apache.pinot.plugin.ingestion.batch.standalone.SegmentTarPushJobRunner&#x27;</span>
</span><span class="code-line"> <span class="token atrule key">segmentUriPushJobRunnerClassName</span><span class="token punctuation">:</span> <span class="token string">&#x27;org.apache.pinot.plugin.ingestion.batch.standalone.SegmentUriPushJobRunner&#x27;</span>
</span><span class="code-line"><span class="token atrule key">jobType</span><span class="token punctuation">:</span> SegmentCreationAndTarPush
</span><span class="code-line"><span class="token atrule key">inputDirURI</span><span class="token punctuation">:</span> <span class="token string">&#x27;s3://marks-st-cloud-bucket/events/&#x27;</span>
</span><span class="code-line"><span class="token atrule key">includeFileNamePattern</span><span class="token punctuation">:</span> <span class="token string">&#x27;glob:**/*.json&#x27;</span>
</span><span class="code-line"><span class="token atrule key">outputDirURI</span><span class="token punctuation">:</span> <span class="token string">&#x27;/data&#x27;</span>
</span><span class="code-line"><span class="token atrule key">overwriteOutput</span><span class="token punctuation">:</span> <span class="token boolean important">true</span>
</span><span class="code-line"><span class="token atrule key">pinotFSSpecs</span><span class="token punctuation">:</span>
</span><span class="code-line"> <span class="token punctuation">-</span> <span class="token atrule key">scheme</span><span class="token punctuation">:</span> s3
</span><span class="code-line"> <span class="token atrule key">className</span><span class="token punctuation">:</span> org.apache.pinot.plugin.filesystem.S3PinotFS
</span><span class="code-line"> <span class="token atrule key">configs</span><span class="token punctuation">:</span>
</span><span class="code-line"> <span class="token atrule key">region</span><span class="token punctuation">:</span> <span class="token string">&#x27;eu-west-2&#x27;</span>
</span><span class="code-line"> <span class="token punctuation">-</span> <span class="token atrule key">scheme</span><span class="token punctuation">:</span> file
</span><span class="code-line"> <span class="token atrule key">className</span><span class="token punctuation">:</span> org.apache.pinot.spi.filesystem.LocalPinotFS
</span><span class="code-line"><span class="token atrule key">recordReaderSpec</span><span class="token punctuation">:</span>
</span><span class="code-line"> <span class="token atrule key">dataFormat</span><span class="token punctuation">:</span> <span class="token string">&#x27;json&#x27;</span>
</span><span class="code-line"> <span class="token atrule key">className</span><span class="token punctuation">:</span> <span class="token string">&#x27;org.apache.pinot.plugin.inputformat.json.JSONRecordReader&#x27;</span>
</span><span class="code-line"><span class="token atrule key">tableSpec</span><span class="token punctuation">:</span>
</span><span class="code-line"> <span class="token atrule key">tableName</span><span class="token punctuation">:</span> <span class="token string">&#x27;events&#x27;</span>
</span><span class="code-line"><span class="token atrule key">pinotClusterSpecs</span><span class="token punctuation">:</span>
</span><span class="code-line"> <span class="token punctuation">-</span> <span class="token atrule key">controllerURI</span><span class="token punctuation">:</span> <span class="token string">&#x27;http://${PINOT_CONTROLLER}:9000&#x27;</span>
</span></code></pre></div><p>At a high level, this file describes a batch import job that will ingest files from the S3 bucket at s3://marks-st-cloud-bucket/events/ where the files match the glob:**/*.json pattern.</p><p>We can import the data by running the following command from the terminal:</p><div class="relative"><pre><code class="code-highlight language-bash"><span class="code-line"><span class="token function">docker</span> run <span class="token punctuation">\</span>
</span><span class="code-line"> <span class="token variable parameter">--network</span> ingest-json-files-s3 <span class="token punctuation">\</span>
</span><span class="code-line"> <span class="token variable parameter">-v</span> <span class="token constant environment">$PWD</span>/config:/config <span class="token punctuation">\</span>
</span><span class="code-line"> <span class="token variable parameter">-e</span> <span class="token variable assign-left">AWS_ACCESS_KEY_ID</span><span class="token operator">=</span>AKIARCOCT6DWLUB7F77Z <span class="token punctuation">\</span>
</span><span class="code-line"> <span class="token variable parameter">-e</span> <span class="token variable assign-left">AWS_SECRET_ACCESS_KEY</span><span class="token operator">=</span>gfz71RX+Tj4udve43YePCBqMsIeN1PvHXrVFyxJS <span class="token punctuation">\</span>
</span><span class="code-line"> apachepinot/pinot:0.11.0 LaunchDataIngestionJob <span class="token punctuation">\</span>
</span><span class="code-line"> <span class="token variable parameter">-jobSpecFile</span> /config/job-spec.yml <span class="token punctuation">\</span>
</span><span class="code-line"> <span class="token variable parameter">-values</span> <span class="token variable assign-left">PINOT_CONTROLLER</span><span class="token operator">=</span>pinot-controller
</span></code></pre></div><p>And don’t worry, those credentials have already been deleted; I find it easier to understand what values go where if we use real values.</p><p>Once we’ve run this command, if we go to the Pinot UI at <a target="_blank" rel="noopener noreferrer" href="http://localhost:9000/">http://localhost:9000</a> and click through to the events table from the Query Console menu, we’ll see that the records have been imported, as shown in the screenshot below:</p><p><img alt="Sample imported records shown in the Apache Pinot Query Console menu" src="https://www.datocms-assets.com/75153/1668701512-image3.png" title="Sample imported records shown in the Apache Pinot Query Console menu"/></p><p>This approach works, and we may still prefer to use it when we need fine-grained control over the ingestion parameters, but it is a bit heavyweight for your everyday data import!</p><h2 id="batch-importing-with-sql"><a href="#batch-importing-with-sql" aria-hidden="true" tabindex="-1"><span class="icon icon-link"></span></a>Batch Importing with SQL</h2><p>Now let’s do the same thing in SQL.</p><p>There are some prerequisites to using the SQL approach, so let’s go through those now, so you don’t end up with a bunch of exceptions when you try this out!</p><p>First of all, you must have a <a target="_blank" rel="noopener noreferrer" href="https://docs.pinot.apache.org/basics/components/minion">Minion</a> in the Pinot cluster, as this is the component that will do the data import.</p><p>You’ll also need to include the following in your table config:</p><div class="relative"><pre><code class="code-highlight language-json"><span class="code-line"><span class="token property">&quot;task&quot;</span><span class="token operator">:</span> <span class="token punctuation">{</span>
</span><span class="code-line"> <span class="token property">&quot;taskTypeConfigsMap&quot;</span><span class="token operator">:</span> <span class="token punctuation">{</span> <span class="token property">&quot;SegmentGenerationAndPushTask&quot;</span><span class="token operator">:</span> <span class="token punctuation">{</span><span class="token punctuation">}</span> <span class="token punctuation">}</span>
</span><span class="code-line"><span class="token punctuation">}</span>
</span></code></pre></div><p>As long as you’ve done those two things, we’re ready to write our import query! A query that imports JSON files from my S3 bucket is shown below:</p><div class="relative"><pre><code class="code-highlight language-sql"><span class="code-line"><span class="token keyword">INSERT</span> <span class="token keyword">INTO</span> events
</span><span class="code-line"><span class="token keyword">FROM</span> <span class="token keyword">FILE</span> <span class="token string">&#x27;s3://marks-st-cloud-bucket/events/&#x27;</span>
</span><span class="code-line"><span class="token keyword">OPTION</span><span class="token punctuation">(</span>
</span><span class="code-line"> taskName<span class="token operator">=</span>events<span class="token operator">-</span>task<span class="token punctuation">,</span>
</span><span class="code-line"> includeFileNamePattern<span class="token operator">=</span>glob:<span class="token operator">*</span><span class="token operator">*</span><span class="token operator">/</span><span class="token operator">*</span><span class="token punctuation">.</span>json<span class="token punctuation">,</span>
</span><span class="code-line"> input<span class="token punctuation">.</span>fs<span class="token punctuation">.</span>className<span class="token operator">=</span>org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>pinot<span class="token punctuation">.</span>plugin<span class="token punctuation">.</span>filesystem<span class="token punctuation">.</span>S3PinotFS<span class="token punctuation">,</span>
</span><span class="code-line"> input<span class="token punctuation">.</span>fs<span class="token punctuation">.</span>prop<span class="token punctuation">.</span>accessKey<span class="token operator">=</span>AKIARCOCT6DWLUB7F77Z<span class="token punctuation">,</span>
</span><span class="code-line"> input<span class="token punctuation">.</span>fs<span class="token punctuation">.</span>prop<span class="token punctuation">.</span>secretKey<span class="token operator">=</span>gfz71RX<span class="token operator">+</span>Tj4udve43YePCBqMsIeN1PvHXrVFyxJS<span class="token punctuation">,</span>
</span><span class="code-line"> input<span class="token punctuation">.</span>fs<span class="token punctuation">.</span>prop<span class="token punctuation">.</span>region<span class="token operator">=</span>eu<span class="token operator">-</span>west<span class="token operator">-</span><span class="token number">2</span>
</span><span class="code-line"><span class="token punctuation">)</span><span class="token punctuation">;</span>
</span></code></pre></div><p>If we run this query, we’ll see the following output:</p><p><img alt="Sample events_OFFLINE query result" src="https://www.datocms-assets.com/75153/1668701654-image5.png" title="Sample events_OFFLINE query result"/></p><p>We can check on the state of the ingestion job via the Swagger REST API. If we navigate to <a target="_blank" rel="noopener noreferrer" href="http://localhost:9000/help#/Task/getTaskState">http://localhost:9000/help#/Task/getTaskState</a>, paste Task_SegmentGenerationAndPushTask_events-task as our task name, and then click Execute, we’ll see the following:</p><p><img alt="Checking the state of an ingestion job screen" src="https://www.datocms-assets.com/75153/1668701727-image2.png" title="Checking the state of an ingestion job screen"/></p><p>If we see the state COMPLETED, this means the data has been ingested, which we can check by going back to the Query console and clicking on the events table.</p><h2 id="summary"><a href="#summary" aria-hidden="true" tabindex="-1"><span class="icon icon-link"></span></a>Summary</h2><p>I have to say that batch ingestion of data into Apache Pinot has always felt a bit clunky, but with this new clause, it’s super easy, and it’s gonna save us all a bunch of time.</p><p>Also, anything that means I’m not writing YAML files has got to be a good thing!</p><p>So give it a try and let us know how you get on. If you have any questions about this feature, feel free to join us on <a target="_blank" rel="noopener noreferrer" href="https://stree.ai/slack">Slack</a>, where we’ll be happy to help you out.</p></div></main></div></div><aside class="mt-10 hidden border-l-2 pl-5 lg:sticky lg:top-1 lg:block lg:h-full"><section class="sticky top-0 mb-4 w-[15.375rem]"><div class="flex flex-col space-y-1.5 pb-3"><h3 class="text-sm font-semibold leading-snug text-neutral-500 dark:text-neutral-100">Table of Contents</h3></div><nav class="flex items-center self-start" aria-label="Table of Contents"><ol class="list-none space-y-3"><li class="text-sm leading-tight font-bold"><a href="#batch-importing-the-job-specification">Batch importing: The Job Specification</a></li><li class="text-sm font-normal leading-tight"><a href="#batch-importing-with-sql">Batch Importing with SQL</a></li><li class="text-sm font-normal leading-tight"><a href="#summary">Summary</a></li></ol></nav></section></aside></div></article></section></main><footer class="border-t bg-sky-100 px-5 py-10 md:px-[6.75rem] md:pb-10 md:pt-16"><div class="mx-auto flex max-w-7xl flex-wrap justify-between"><div class="flex-shrink-0"><svg xmlns="http://www.w3.org/2000/svg" width="120" height="48" fill="none"><g fill="#C7154A" clip-path="url(#logo_svg__a)"><path d="M42.99 18.448c1.032-.553 2.21-.831 3.535-.831 1.542 0 2.938.38 4.187 1.14 1.248.76 2.236 1.841 2.965 3.241.728 1.402 1.091 3.025 1.091 4.872s-.363 3.482-1.091 4.903c-.729 1.424-1.717 2.525-2.965 3.307-1.25.782-2.645 1.173-4.187 1.173-1.325 0-2.493-.271-3.503-.815-1.01-.543-1.83-1.226-2.46-2.053v14.612H36V17.912h4.562v2.606c.586-.825 1.395-1.515 2.426-2.068l.002-.002m6.452 5.605c-.445-.793-1.032-1.395-1.76-1.808a4.72 4.72 0 0 0-2.362-.618c-.847 0-1.602.211-2.33.635-.728.423-1.315 1.038-1.76 1.841-.445.804-.668 1.749-.668 2.835 0 1.087.221 2.032.668 2.835.445.804 1.032 1.417 1.76 1.842a4.557 4.557 0 0 0 2.33.635 4.57 4.57 0 0 0 2.362-.652c.728-.435 1.313-1.053 1.76-1.856.445-.804.668-1.76.668-2.867s-.223-2.025-.668-2.818v-.004M62.947 17.912v18.051h-4.562V17.912h4.562m.551-6.079a2.833 2.833 0 1 1-5.666 0 2.833 2.833 0 0 1 5.666 0M82.954 19.687c1.325 1.358 1.988 3.253 1.988 5.685v10.59H80.38v-9.97c0-1.434-.358-2.537-1.075-3.307-.717-.772-1.695-1.157-2.933-1.157-1.239 0-2.254.387-2.982 1.157-.728.772-1.091 1.873-1.091 3.307v9.97h-4.562V17.91h4.562v2.248a6.322 6.322 0 0 1 2.33-1.841c.944-.445 1.981-.669 3.111-.669 2.15 0 3.889.68 5.214 2.037v.002M92.892 35.098c-1.39-.77-2.482-1.861-3.275-3.275-.794-1.411-1.19-3.041-1.19-4.888s.406-3.475 1.221-4.888a8.502 8.502 0 0 1 3.34-3.275c1.412-.772 2.987-1.157 4.725-1.157 1.739 0 3.312.387 4.725 1.157a8.5 8.5 0 0 1 3.34 3.275c.815 1.411 1.222 3.041 1.222 4.888s-.418 3.475-1.255 4.888a8.708 8.708 0 0 1-3.388 3.275c-1.424.772-3.014 1.157-4.774 1.157-1.76 0-3.301-.385-4.691-1.157m7.021-3.421c.729-.402 1.309-1.005 1.744-1.809.435-.803.651-1.781.651-2.933 0-1.715-.451-3.035-1.351-3.958-.902-.924-2.004-1.385-3.307-1.385s-2.395.461-3.275 1.385c-.88.923-1.32 2.243-1.32 3.958 0 1.715.428 3.035 1.287 3.958.858.924 1.938 1.385 3.241 1.385.825 0 1.602-.2 2.33-.603v.002M115.96 21.658v8.734c0 .608.147 1.048.44 1.32.293.271.787.406 1.482.406H120v3.845h-2.867c-3.845 0-5.766-1.868-5.766-5.605v-8.7h-2.15v-3.746h2.15V13l4.595-1v5.912h4.04v3.746h-4.042M20.03 46.757l-5.538-1.385A1.97 1.97 0 0 1 13 43.46v-5.462c0-.841.349-1.601.907-2.146a12.212 12.212 0 0 0 6.975-3.644c2.602-2.731 3.627-6.578 2.882-10.251L21 9h-4V4a1 1 0 0 0-2 0v7a1 1 0 0 1-2 0v-1a1 1 0 0 0-2 0v6.758a4.489 4.489 0 0 1 2.694-.755c2.278.095 4.156 1.934 4.297 4.21a4.501 4.501 0 0 1-6.992 4.029V29a1 1 0 0 1-2 0V7a1 1 0 0 0-2 0v2h-4L.237 21.957c-.745 3.675.279 7.52 2.882 10.251a12.202 12.202 0 0 0 6.975 3.644c.558.545.907 1.305.907 2.146V43.4c0 .938-.639 1.757-1.55 1.985l-5.48 1.37c-.57.143-.97.655-.97 1.243h18c0-.588-.4-1.1-.97-1.243v.002"></path><path d="M13.5 23a2.5 2.5 0 1 0 0-5 2.5 2.5 0 0 0 0 5M8 5a1 1 0 1 0 0-2 1 1 0 0 0 0 2M12 8a1 1 0 1 0 0-2 1 1 0 0 0 0 2M16 2a1 1 0 1 0 0-2 1 1 0 0 0 0 2"></path></g><defs><clipPath id="logo_svg__a"><path fill="#fff" d="M0 0h120v48H0z"></path></clipPath></defs></svg></div><div class="flex flex-wrap gap-x-16 gap-y-5 py-8 md:pl-24 md:pr-[21.625rem]"> <div><h5 class="mb-4 text-lg font-semibold">Resources</h5><div class="flex justify-between gap-x-10"><div class="flex flex-col"><a target="_blank" rel="noopener noreferrer" href="https://docs.pinot.apache.org/" class="block py-1 text-gray-600 hover:text-gray-900">Docs</a><a target="_blank" rel="noopener noreferrer" href="https://docs.pinot.apache.org/getting-started" class="block py-1 text-gray-600 hover:text-gray-900">Getting Started</a><a target="_blank" rel="noopener noreferrer" href="https://docs.pinot.apache.org/integrations/thirdeye" class="block py-1 text-gray-600 hover:text-gray-900">ThirdEye</a></div><div class="flex flex-col"><a class="block py-1 text-gray-600 hover:text-gray-900" href="/powered-by">Company Stories</a><a class="block py-1 text-gray-600 hover:text-gray-900" href="/download">Download</a><a class="block py-1 text-gray-600 hover:text-gray-900" href="/blog">Blog</a></div></div></div><div><h5 class="mb-4 text-lg font-semibold">Apache</h5><div class="flex justify-between gap-x-10"><div class="flex flex-col"><a target="_blank" rel="noopener noreferrer" href="https://www.apache.org" class="block py-1 text-gray-600 hover:text-gray-900">Foundation</a><a target="_blank" rel="noopener noreferrer" href="https://www.apache.org/licenses" class="block py-1 text-gray-600 hover:text-gray-900">License</a><a target="_blank" rel="noopener noreferrer" href="https://www.apache.org/security" class="block py-1 text-gray-600 hover:text-gray-900">Security</a></div><div class="flex flex-col"><a target="_blank" rel="noopener noreferrer" href="https://www.apache.org/foundation/sponsorship.html" class="block py-1 text-gray-600 hover:text-gray-900">Sponsorship</a><a target="_blank" rel="noopener noreferrer" href="https://www.apache.org/events/current-event" class="block py-1 text-gray-600 hover:text-gray-900">Events</a><a target="_blank" rel="noopener noreferrer" href="https://www.apache.org/foundation/thanks.html" class="block py-1 text-gray-600 hover:text-gray-900">Thanks</a></div></div></div></div><div class="mt-4 flex justify-center md:mt-0"><a target="_blank" rel="noopener noreferrer" href="https://join.slack.com/t/apache-pinot/shared_invite/zt-5z7pav2f-yYtjZdVA~EDmrGkho87Vzw" class="mr-4"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-slack fill-gray-900"><rect width="3" height="8" x="13" y="2" rx="1.5"></rect><path d="M19 8.5V10h1.5A1.5 1.5 0 1 0 19 8.5"></path><rect width="3" height="8" x="8" y="14" rx="1.5"></rect><path d="M5 15.5V14H3.5A1.5 1.5 0 1 0 5 15.5"></path><rect width="8" height="3" x="14" y="13" rx="1.5"></rect><path d="M15.5 19H14v1.5a1.5 1.5 0 1 0 1.5-1.5"></path><rect width="8" height="3" x="2" y="8" rx="1.5"></rect><path d="M8.5 5H10V3.5A1.5 1.5 0 1 0 8.5 5"></path></svg></a><a target="_blank" rel="noopener noreferrer" href="https://github.com/apache/pinot"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="currentColor" size="24"><g clip-path="url(#github_svg__a)"><path fill-rule="evenodd" d="M12.01 0C5.369 0 0 5.5 0 12.304c0 5.44 3.44 10.043 8.212 11.673.597.122.815-.265.815-.59 0-.286-.02-1.264-.02-2.283-3.34.734-4.036-1.466-4.036-1.466-.537-1.426-1.332-1.793-1.332-1.793-1.094-.754.08-.754.08-.754 1.212.082 1.849 1.263 1.849 1.263 1.073 1.874 2.803 1.345 3.5 1.019.098-.795.417-1.345.755-1.65-2.665-.285-5.468-1.345-5.468-6.07 0-1.345.477-2.445 1.232-3.3-.119-.306-.537-1.57.12-3.26 0 0 1.014-.326 3.3 1.263.98-.27 1.989-.407 3.003-.408 1.014 0 2.048.143 3.002.408 2.287-1.59 3.301-1.263 3.301-1.263.657 1.69.239 2.954.12 3.26.775.855 1.232 1.955 1.232 3.3 0 4.725-2.803 5.764-5.488 6.07.438.387.815 1.12.815 2.281 0 1.65-.02 2.975-.02 3.382 0 .326.22.713.816.59C20.56 22.347 24 17.744 24 12.305 24.02 5.5 18.63 0 12.01 0" clip-rule="evenodd"></path></g><defs><clipPath id="github_svg__a"><path fill="#fff" d="M0 0h24v24H0z"></path></clipPath></defs></svg></a></div></div><div class="mt-8 border-t border-neutral-300 pt-4 text-left text-sm text-gray-600">Copyright © <!-- -->2024<!-- --> The Apache Software Foundation. Apache Pinot, Pinot, Apache, the Apache feather logo, and the Apache Pinot project logo are registered trademarks of The Apache Software Foundation. This page has references to third party software - Presto, PrestoDB, ThirdEye, Trino, TrinoDB, that are not part of the Apache Software Foundation and are not covered under the Apache License.</div></footer></div><script src="/_next/static/chunks/webpack-dde39ac7c1b4eb4b.js" crossorigin="" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/_next/static/media/6905431624c34d00-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/_next/static/css/9e925a33b1acdac1.css\",\"style\",{\"crossOrigin\":\"\"}]\n0:\"$L3\"\n"])</script><script>self.__next_f.push([1,"4:HL[\"/_next/static/css/c130d1629644f070.css\",\"style\",{\"crossOrigin\":\"\"}]\n"])</script><script>self.__next_f.push([1,"5:I[3728,[],\"\"]\n7:I[9928,[],\"\"]\n8:I[7821,[\"326\",\"static/chunks/326-3a90a6443b9c824c.js\",\"980\",\"static/chunks/980-6e243f9cd384c7d2.js\",\"702\",\"static/chunks/702-a2bf9fe707814b79.js\",\"185\",\"static/chunks/app/layout-776a485845c720ef.js\"],\"ThemeProviders\"]\n9:I[3994,[\"326\",\"static/chunks/326-3a90a6443b9c824c.js\",\"980\",\"static/chunks/980-6e243f9cd384c7d2.js\",\"702\",\"static/chunks/702-a2bf9fe707814b79.js\",\"185\",\"static/chunks/app/layout-776a485845c720ef.js\"],\"\"]\na:I[9640,[\"326\",\"static/chunks/326-3a90a6443b9c824c.js"])</script><script>self.__next_f.push([1,"\",\"980\",\"static/chunks/980-6e243f9cd384c7d2.js\",\"702\",\"static/chunks/702-a2bf9fe707814b79.js\",\"185\",\"static/chunks/app/layout-776a485845c720ef.js\"],\"AlgoliaSearchProvider\"]\nb:I[7975,[\"326\",\"static/chunks/326-3a90a6443b9c824c.js\",\"980\",\"static/chunks/980-6e243f9cd384c7d2.js\",\"702\",\"static/chunks/702-a2bf9fe707814b79.js\",\"185\",\"static/chunks/app/layout-776a485845c720ef.js\"],\"\"]\nc:I[6954,[],\"\"]\nd:I[7264,[],\"\"]\ne:I[8326,[\"326\",\"static/chunks/326-3a90a6443b9c824c.js\",\"413\",\"static/chunks/413-f9f40b83f7bb3f22.js\""])</script><script>self.__next_f.push([1,",\"980\",\"static/chunks/980-6e243f9cd384c7d2.js\",\"797\",\"static/chunks/app/blog/%5B...slug%5D/page-502e08b6677b55da.js\"],\"\"]\n11:T9fe,"])</script><script>self.__next_f.push([1,"M42.99 18.448c1.032-.553 2.21-.831 3.535-.831 1.542 0 2.938.38 4.187 1.14 1.248.76 2.236 1.841 2.965 3.241.728 1.402 1.091 3.025 1.091 4.872s-.363 3.482-1.091 4.903c-.729 1.424-1.717 2.525-2.965 3.307-1.25.782-2.645 1.173-4.187 1.173-1.325 0-2.493-.271-3.503-.815-1.01-.543-1.83-1.226-2.46-2.053v14.612H36V17.912h4.562v2.606c.586-.825 1.395-1.515 2.426-2.068l.002-.002m6.452 5.605c-.445-.793-1.032-1.395-1.76-1.808a4.72 4.72 0 0 0-2.362-.618c-.847 0-1.602.211-2.33.635-.728.423-1.315 1.038-1.76 1.841-.445.804-.668 1.749-.668 2.835 0 1.087.221 2.032.668 2.835.445.804 1.032 1.417 1.76 1.842a4.557 4.557 0 0 0 2.33.635 4.57 4.57 0 0 0 2.362-.652c.728-.435 1.313-1.053 1.76-1.856.445-.804.668-1.76.668-2.867s-.223-2.025-.668-2.818v-.004M62.947 17.912v18.051h-4.562V17.912h4.562m.551-6.079a2.833 2.833 0 1 1-5.666 0 2.833 2.833 0 0 1 5.666 0M82.954 19.687c1.325 1.358 1.988 3.253 1.988 5.685v10.59H80.38v-9.97c0-1.434-.358-2.537-1.075-3.307-.717-.772-1.695-1.157-2.933-1.157-1.239 0-2.254.387-2.982 1.157-.728.772-1.091 1.873-1.091 3.307v9.97h-4.562V17.91h4.562v2.248a6.322 6.322 0 0 1 2.33-1.841c.944-.445 1.981-.669 3.111-.669 2.15 0 3.889.68 5.214 2.037v.002M92.892 35.098c-1.39-.77-2.482-1.861-3.275-3.275-.794-1.411-1.19-3.041-1.19-4.888s.406-3.475 1.221-4.888a8.502 8.502 0 0 1 3.34-3.275c1.412-.772 2.987-1.157 4.725-1.157 1.739 0 3.312.387 4.725 1.157a8.5 8.5 0 0 1 3.34 3.275c.815 1.411 1.222 3.041 1.222 4.888s-.418 3.475-1.255 4.888a8.708 8.708 0 0 1-3.388 3.275c-1.424.772-3.014 1.157-4.774 1.157-1.76 0-3.301-.385-4.691-1.157m7.021-3.421c.729-.402 1.309-1.005 1.744-1.809.435-.803.651-1.781.651-2.933 0-1.715-.451-3.035-1.351-3.958-.902-.924-2.004-1.385-3.307-1.385s-2.395.461-3.275 1.385c-.88.923-1.32 2.243-1.32 3.958 0 1.715.428 3.035 1.287 3.958.858.924 1.938 1.385 3.241 1.385.825 0 1.602-.2 2.33-.603v.002M115.96 21.658v8.734c0 .608.147 1.048.44 1.32.293.271.787.406 1.482.406H120v3.845h-2.867c-3.845 0-5.766-1.868-5.766-5.605v-8.7h-2.15v-3.746h2.15V13l4.595-1v5.912h4.04v3.746h-4.042M20.03 46.757l-5.538-1.385A1.97 1.97 0 0 1 13 43.46v-5.462c0-.841.349-1.601.907-2.146a12.212 12.212 0 0 0 6.975-3.644c2.602-2.731 3.627-6.578 2.882-10.251L21 9h-4V4a1 1 0 0 0-2 0v7a1 1 0 0 1-2 0v-1a1 1 0 0 0-2 0v6.758a4.489 4.489 0 0 1 2.694-.755c2.278.095 4.156 1.934 4.297 4.21a4.501 4.501 0 0 1-6.992 4.029V29a1 1 0 0 1-2 0V7a1 1 0 0 0-2 0v2h-4L.237 21.957c-.745 3.675.279 7.52 2.882 10.251a12.202 12.202 0 0 0 6.975 3.644c.558.545.907 1.305.907 2.146V43.4c0 .938-.639 1.757-1.55 1.985l-5.48 1.37c-.57.143-.97.655-.97 1.243h18c0-.588-.4-1.1-.97-1.243v.002"])</script><script>self.__next_f.push([1,"3:[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/9e925a33b1acdac1.css\",\"precedence\":\"next\",\"crossOrigin\":\"\"}]],[\"$\",\"$L5\",null,{\"buildId\":\"rmcKjFZ3e9kKdH1iJwCIQ\",\"assetPrefix\":\"\",\"initialCanonicalUrl\":\"/blog/2022/11/17/Apache%20Pinot-Inserts-from-SQL\",\"initialTree\":[\"\",{\"children\":[\"blog\",{\"children\":[[\"slug\",\"2022/11/17/Apache%20Pinot-Inserts-from-SQL\",\"c\"],{\"children\":[\"__PAGE__?{\\\"slug\\\":[\\\"2022\\\",\\\"11\\\",\\\"17\\\",\\\"Apache Pinot-Inserts-from-SQL\\\"]}\",{}]}]}]},\"$undefined\",\"$undefined\",true],\"initialHead\":[false,\"$L6\"],\"globalErrorComponent\":\"$7\",\"children\":[null,[\"$\",\"html\",null,{\"lang\":\"en-us\",\"className\":\"__variable_1fc36d scroll-smooth\",\"suppressHydrationWarning\":true,\"children\":[[\"$\",\"head\",null,{\"children\":[[\"$\",\"meta\",null,{\"httpEquiv\":\"Content-Security-Policy\",\"content\":\"default-src 'self';script-src 'self' 'unsafe-eval' 'unsafe-inline' giscus.app analytics.umami.is www.youtube.com www.googletagmanager.com www.google-analytics.com;style-src 'self' 'unsafe-inline';img-src * blob: data:;media-src *.s3.amazonaws.com;connect-src *;font-src 'self';frame-src www.youtube.com youtube.com giscus.app youtu.be https://www.youtube.com https://youtube.com;\"}],[\"$\",\"link\",null,{\"rel\":\"apple-touch-icon\",\"sizes\":\"76x76\",\"href\":\"/static/favicons/apple-touch-icon.png\"}],[\"$\",\"link\",null,{\"rel\":\"icon\",\"type\":\"image/png\",\"sizes\":\"32x32\",\"href\":\"/static/favicons/favicon-32x32.png\"}],[\"$\",\"link\",null,{\"rel\":\"icon\",\"type\":\"image/png\",\"sizes\":\"16x16\",\"href\":\"/static/favicons/favicon-16x16.png\"}],[\"$\",\"link\",null,{\"rel\":\"manifest\",\"href\":\"/static/favicons/site.webmanifest\"}],[\"$\",\"link\",null,{\"rel\":\"mask-icon\",\"href\":\"/static/favicons/safari-pinned-tab.svg\",\"color\":\"#5bbad5\"}],[\"$\",\"meta\",null,{\"name\":\"msapplication-TileColor\",\"content\":\"#000000\"}],[\"$\",\"meta\",null,{\"name\":\"theme-color\",\"media\":\"(prefers-color-scheme: light)\",\"content\":\"#fff\"}],[\"$\",\"meta\",null,{\"name\":\"theme-color\",\"media\":\"(prefers-color-scheme: dark)\",\"content\":\"#000\"}],[\"$\",\"link\",null,{\"rel\":\"alternate\",\"type\":\"application/rss+xml\",\"href\":\"/feed.xml\"}]]}],[\"$\",\"body\",null,{\"className\":\"bg-white text-black antialiased dark:bg-gray-950 dark:text-white\",\"children\":[\"$\",\"$L8\",null,{\"children\":[[\"$undefined\",\"$undefined\",\"$undefined\",\"$undefined\",[[\"$\",\"$L9\",null,{\"strategy\":\"afterInteractive\",\"src\":\"https://www.googletagmanager.com/gtag/js?id=G-ZXG79NJEBY\"}],[\"$\",\"$L9\",null,{\"strategy\":\"afterInteractive\",\"id\":\"ga-script\",\"children\":\"\\n window.dataLayer = window.dataLayer || [];\\n function gtag(){dataLayer.push(arguments);}\\n gtag('js', new Date());\\n gtag('config', 'G-ZXG79NJEBY');\\n \"}]]],[\"$\",\"div\",null,{\"className\":\"mx-auto flex max-w-screen-customDesktop flex-col justify-between font-sans\",\"children\":[\"$\",\"$La\",null,{\"algoliaConfig\":{\"appId\":\"CKRA00L2X9\",\"apiKey\":\"6531f8f7783a88d76629190843f1801e\",\"indexName\":\"prod_apache_pinot_docs\"},\"children\":[[\"$\",\"$Lb\",null,{}],[\"$\",\"main\",null,{\"children\":[\"$\",\"$Lc\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"loadingScripts\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$Ld\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[\"$\",\"div\",null,{\"className\":\"flex flex-col items-start justify-start md:mt-24 md:flex-row md:items-center md:justify-center md:space-x-6\",\"children\":[[\"$\",\"div\",null,{\"className\":\"space-x-2 pb-8 pt-6 md:space-y-5\",\"children\":[\"$\",\"h1\",null,{\"className\":\"text-6xl font-extrabold leading-9 tracking-tight text-gray-900 dark:text-gray-100 md:border-r-2 md:px-6 md:text-8xl md:leading-14\",\"children\":\"404\"}]}],[\"$\",\"div\",null,{\"className\":\"max-w-md\",\"children\":[[\"$\",\"p\",null,{\"className\":\"mb-4 text-xl font-bold leading-normal md:text-2xl\",\"children\":\"Sorry we couldn't find this page.\"}],[\"$\",\"p\",null,{\"className\":\"mb-8\",\"children\":\"But dont worry, you can find plenty of other things on our homepage.\"}],[\"$\",\"$Le\",null,{\"href\":\"/\",\"className\":\"focus:shadow-outline-blue inline rounded-lg border border-transparent bg-blue-600 px-4 py-2 text-sm font-medium leading-5 text-white shadow transition-colors duration-150 hover:bg-blue-700 focus:outline-none dark:hover:bg-blue-500\",\"children\":\"Back to homepage\"}]]}]]}],\"notFoundStyles\":[],\"initialChildNode\":[\"$\",\"$Lc\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\",\"blog\",\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"loadingScripts\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$Ld\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"notFoundStyles\":\"$undefined\",\"initialChildNode\":[\"$\",\"$Lc\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\",\"blog\",\"children\",[\"slug\",\"2022/11/17/Apache%20Pinot-Inserts-from-SQL\",\"c\"],\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"loadingScripts\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$Ld\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"notFoundStyles\":\"$undefined\",\"initialChildNode\":[\"$Lf\",\"$L10\",null],\"childPropSegment\":\"__PAGE__?{\\\"slug\\\":[\\\"2022\\\",\\\"11\\\",\\\"17\\\",\\\"Apache Pinot-Inserts-from-SQL\\\"]}\",\"styles\":[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/c130d1629644f070.css\",\"precedence\":\"next\",\"crossOrigin\":\"\"}]]}],\"childPropSegment\":[\"slug\",\"2022/11/17/Apache%20Pinot-Inserts-from-SQL\",\"c\"],\"styles\":null}],\"childPropSegment\":\"blog\",\"styles\":null}]}],[\"$\",\"footer\",null,{\"className\":\"border-t bg-sky-100 px-5 py-10 md:px-[6.75rem] md:pb-10 md:pt-16\",\"children\":[[\"$\",\"div\",null,{\"className\":\"mx-auto flex max-w-7xl flex-wrap justify-between\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex-shrink-0\",\"children\":[\"$\",\"svg\",null,{\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":120,\"height\":48,\"fill\":\"none\",\"children\":[[\"$\",\"g\",null,{\"fill\":\"#C7154A\",\"clipPath\":\"url(#logo_svg__a)\",\"children\":[[\"$\",\"path\",null,{\"d\":\"$11\"}],[\"$\",\"path\",null,{\"d\":\"M13.5 23a2.5 2.5 0 1 0 0-5 2.5 2.5 0 0 0 0 5M8 5a1 1 0 1 0 0-2 1 1 0 0 0 0 2M12 8a1 1 0 1 0 0-2 1 1 0 0 0 0 2M16 2a1 1 0 1 0 0-2 1 1 0 0 0 0 2\"}]]}],[\"$\",\"defs\",null,{\"children\":[\"$\",\"clipPath\",null,{\"id\":\"logo_svg__a\",\"children\":[\"$\",\"path\",null,{\"fill\":\"#fff\",\"d\":\"M0 0h120v48H0z\"}]}]}]]}]}],[\"$\",\"div\",null,{\"className\":\"flex flex-wrap gap-x-16 gap-y-5 py-8 md:pl-24 md:pr-[21.625rem]\",\"children\":[\" \",[[\"$\",\"div\",\"Resources\",{\"children\":[[\"$\",\"h5\",null,{\"className\":\"mb-4 text-lg font-semibold\",\"children\":\"Resources\"}],[\"$\",\"div\",null,{\"className\":\"flex justify-between gap-x-10\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex flex-col\",\"children\":[[\"$\",\"a\",null,{\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"href\":\"https://docs.pinot.apache.org/\",\"className\":\"block py-1 text-gray-600 hover:text-gray-900\",\"children\":\"Docs\"}],[\"$\",\"a\",null,{\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"href\":\"https://docs.pinot.apache.org/getting-started\",\"className\":\"block py-1 text-gray-600 hover:text-gray-900\",\"children\":\"Getting Started\"}],[\"$\",\"a\",null,{\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"href\":\"https://docs.pinot.apache.org/integrations/thirdeye\",\"className\":\"block py-1 text-gray-600 hover:text-gray-900\",\"children\":\"ThirdEye\"}]]}],[\"$\",\"div\",null,{\"className\":\"flex flex-col\",\"children\":[[\"$\",\"$Le\",null,{\"href\":\"/powered-by\",\"className\":\"block py-1 text-gray-600 hover:text-gray-900\",\"children\":\"Company Stories\"}],[\"$\",\"$Le\",null,{\"href\":\"/download\",\"className\":\"block py-1 text-gray-600 hover:text-gray-900\",\"children\":\"Download\"}],[\"$\",\"$Le\",null,{\"href\":\"/blog\",\"className\":\"block py-1 text-gray-600 hover:text-gray-900\",\"children\":\"Blog\"}]]}]]}]]}],[\"$\",\"div\",\"Apache\",{\"children\":[[\"$\",\"h5\",null,{\"className\":\"mb-4 text-lg font-semibold\",\"children\":\"Apache\"}],[\"$\",\"div\",null,{\"className\":\"flex justify-between gap-x-10\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex flex-col\",\"children\":[[\"$\",\"a\",null,{\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"href\":\"https://www.apache.org\",\"className\":\"block py-1 text-gray-600 hover:text-gray-900\",\"children\":\"Foundation\"}],[\"$\",\"a\",null,{\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"href\":\"https://www.apache.org/licenses\",\"className\":\"block py-1 text-gray-600 hover:text-gray-900\",\"children\":\"License\"}],[\"$\",\"a\",null,{\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"href\":\"https://www.apache.org/security\",\"className\":\"block py-1 text-gray-600 hover:text-gray-900\",\"children\":\"Security\"}]]}],[\"$\",\"div\",null,{\"className\":\"flex flex-col\",\"children\":[[\"$\",\"a\",null,{\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"href\":\"https://www.apache.org/foundation/sponsorship.html\",\"className\":\"block py-1 text-gray-600 hover:text-gray-900\",\"children\":\"Sponsorship\"}],[\"$\",\"a\",null,{\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"href\":\"https://www.apache.org/events/current-event\",\"className\":\"block py-1 text-gray-600 hover:text-gray-900\",\"children\":\"Events\"}],[\"$\",\"a\",null,{\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"href\":\"https://www.apache.org/foundation/thanks.html\",\"className\":\"block py-1 text-gray-600 hover:text-gray-900\",\"children\":\"Thanks\"}]]}]]}]]}]]]}],[\"$\",\"div\",null,{\"className\":\"mt-4 flex justify-center md:mt-0\",\"children\":[[\"$\",\"a\",null,{\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"href\":\"https://join.slack.com/t/apache-pinot/shared_invite/zt-5z7pav2f-yYtjZdVA~EDmrGkho87Vzw\",\"className\":\"mr-4\",\"children\":[\"$\",\"svg\",null,{\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-slack fill-gray-900\",\"children\":[[\"$\",\"rect\",\"diqz80\",{\"width\":\"3\",\"height\":\"8\",\"x\":\"13\",\"y\":\"2\",\"rx\":\"1.5\"}],[\"$\",\"path\",\"183iwg\",{\"d\":\"M19 8.5V10h1.5A1.5 1.5 0 1 0 19 8.5\"}],[\"$\",\"rect\",\"hqg7r1\",{\"width\":\"3\",\"height\":\"8\",\"x\":\"8\",\"y\":\"14\",\"rx\":\"1.5\"}],[\"$\",\"path\",\"76g71w\",{\"d\":\"M5 15.5V14H3.5A1.5 1.5 0 1 0 5 15.5\"}],[\"$\",\"rect\",\"1kmz0a\",{\"width\":\"8\",\"height\":\"3\",\"x\":\"14\",\"y\":\"13\",\"rx\":\"1.5\"}],[\"$\",\"path\",\"jc4sz0\",{\"d\":\"M15.5 19H14v1.5a1.5 1.5 0 1 0 1.5-1.5\"}],[\"$\",\"rect\",\"1omvl4\",{\"width\":\"8\",\"height\":\"3\",\"x\":\"2\",\"y\":\"8\",\"rx\":\"1.5\"}],[\"$\",\"path\",\"16f3cl\",{\"d\":\"M8.5 5H10V3.5A1.5 1.5 0 1 0 8.5 5\"}],\"$undefined\"]}]}],[\"$\",\"a\",null,{\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"href\":\"https://github.com/apache/pinot\",\"children\":[\"$\",\"svg\",null,{\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"fill\":\"currentColor\",\"size\":24,\"children\":[[\"$\",\"g\",null,{\"clipPath\":\"url(#github_svg__a)\",\"children\":[\"$\",\"path\",null,{\"fillRule\":\"evenodd\",\"d\":\"M12.01 0C5.369 0 0 5.5 0 12.304c0 5.44 3.44 10.043 8.212 11.673.597.122.815-.265.815-.59 0-.286-.02-1.264-.02-2.283-3.34.734-4.036-1.466-4.036-1.466-.537-1.426-1.332-1.793-1.332-1.793-1.094-.754.08-.754.08-.754 1.212.082 1.849 1.263 1.849 1.263 1.073 1.874 2.803 1.345 3.5 1.019.098-.795.417-1.345.755-1.65-2.665-.285-5.468-1.345-5.468-6.07 0-1.345.477-2.445 1.232-3.3-.119-.306-.537-1.57.12-3.26 0 0 1.014-.326 3.3 1.263.98-.27 1.989-.407 3.003-.408 1.014 0 2.048.143 3.002.408 2.287-1.59 3.301-1.263 3.301-1.263.657 1.69.239 2.954.12 3.26.775.855 1.232 1.955 1.232 3.3 0 4.725-2.803 5.764-5.488 6.07.438.387.815 1.12.815 2.281 0 1.65-.02 2.975-.02 3.382 0 .326.22.713.816.59C20.56 22.347 24 17.744 24 12.305 24.02 5.5 18.63 0 12.01 0\",\"clipRule\":\"evenodd\"}]}],[\"$\",\"defs\",null,{\"children\":[\"$\",\"clipPath\",null,{\"id\":\"github_svg__a\",\"children\":[\"$\",\"path\",null,{\"fill\":\"#fff\",\"d\":\"M0 0h24v24H0z\"}]}]}]]}]}]]}]]}],[\"$\",\"div\",null,{\"className\":\"mt-8 border-t border-neutral-300 pt-4 text-left text-sm text-gray-600\",\"children\":[\"Copyright © \",2024,\" The Apache Software Foundation. Apache Pinot, Pinot, Apache, the Apache feather logo, and the Apache Pinot project logo are registered trademarks of The Apache Software Foundation. This page has references to third party software - Presto, PrestoDB, ThirdEye, Trino, TrinoDB, that are not part of the Apache Software Foundation and are not covered under the Apache License.\"]}]]}]]}]}]]}]}]]}],null]}]]\n"])</script><script>self.__next_f.push([1,"12:I[1514,[\"326\",\"static/chunks/326-3a90a6443b9c824c.js\",\"413\",\"static/chunks/413-f9f40b83f7bb3f22.js\",\"980\",\"static/chunks/980-6e243f9cd384c7d2.js\",\"797\",\"static/chunks/app/blog/%5B...slug%5D/page-502e08b6677b55da.js\"],\"\"]\n13:I[2529,[\"326\",\"static/chunks/326-3a90a6443b9c824c.js\",\"413\",\"static/chunks/413-f9f40b83f7bb3f22.js\",\"980\",\"static/chunks/980-6e243f9cd384c7d2.js\",\"797\",\"static/chunks/app/blog/%5B...slug%5D/page-502e08b6677b55da.js\"],\"\"]\n14:I[5185,[\"326\",\"static/chunks/326-3a90a6443b9c824c.js\",\"413\",\""])</script><script>self.__next_f.push([1,"static/chunks/413-f9f40b83f7bb3f22.js\",\"980\",\"static/chunks/980-6e243f9cd384c7d2.js\",\"797\",\"static/chunks/app/blog/%5B...slug%5D/page-502e08b6677b55da.js\"],\"\"]\n"])</script><script>self.__next_f.push([1,"10:[[\"$\",\"script\",null,{\"type\":\"application/ld+json\",\"dangerouslySetInnerHTML\":{\"__html\":\"{\\\"@context\\\":\\\"https://schema.org\\\",\\\"@type\\\":\\\"BlogPosting\\\",\\\"headline\\\":\\\"Apache Pinot™ 0.11 - Inserts from SQL\\\",\\\"datePublished\\\":\\\"2022-11-17T00:00:00.000Z\\\",\\\"dateModified\\\":\\\"2022-11-17T00:00:00.000Z\\\",\\\"description\\\":\\\"Explore the INSERT INTO clause, which makes ingesting batch data into Pinot as easy as writing a SQL query.\\\",\\\"image\\\":\\\"/static/images/twitter-card.png\\\",\\\"url\\\":\\\"https://pinot.apache.org/blog/2022-11-17-Apache Pinot-Inserts-from-SQL\\\",\\\"author\\\":[{\\\"@type\\\":\\\"Person\\\",\\\"name\\\":\\\"Mark Needham\\\"}]}\"}}],[\"$\",\"section\",null,{\"className\":\" px-5 pt-10 md:px-[13.313rem] md:py-16\",\"children\":[[\"$\",\"$L12\",null,{}],[\"$\",\"article\",null,{\"className\":\"\",\"children\":[\"$\",\"div\",null,{\"className\":\"mx-auto lg:flex\",\"children\":[[\"$\",\"div\",null,{\"className\":\"lg:pr-12\",\"children\":[[\"$\",\"header\",null,{\"className\":\"pt-6 md:pr-10\",\"children\":[[\"$\",\"h1\",null,{\"className\":\"text-4xl font-semibold\",\"children\":\"Apache Pinot™ 0.11 - Inserts from SQL\"}],[\"$\",\"p\",null,{\"className\":\"pt-2 text-lg\",\"children\":[\"By: \",\"Mark Needham\"]}],[\"$\",\"p\",null,{\"className\":\"py-2 text-sm\",\"children\":[\"November 17th, 2022\",\" • \",\"4 min read\"]}]]}],[\"$\",\"div\",null,{\"className\":\"flex flex-col lg:flex-row\",\"children\":[\"$\",\"main\",null,{\"className\":\"\",\"children\":[\"$\",\"div\",null,{\"className\":\"prose max-w-[45rem] pb-8 pt-10 dark:prose-invert\",\"children\":[[\"$\",\"p\",null,{\"children\":[\"The Apache Pinot community recently released version \",[\"$\",\"a\",null,{\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"href\":\"https://medium.com/apache-pinot-developer-blog/apache-pinot-0-11-released-d564684df5d4\",\"children\":\"0.11.0\"}],\", which has lots of goodies for you to play with. This is the second in a series of blog posts showing off some of the new features in this release.\"]}],[\"$\",\"p\",null,{\"children\":[\"In this post, we’re going to explore the \",[\"$\",\"a\",null,{\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"href\":\"https://docs.pinot.apache.org/basics/data-import/from-query-console\",\"children\":\"INSERT INTO clause\"}],\", which makes ingesting batch data into Pinot as easy as writing a SQL query.\"]}],[\"$\",\"h2\",null,{\"id\":\"batch-importing-the-job-specification\",\"children\":[[\"$\",\"a\",null,{\"href\":\"#batch-importing-the-job-specification\",\"aria-hidden\":\"true\",\"tabIndex\":\"-1\",\"children\":[\"$\",\"span\",null,{\"className\":\"icon icon-link\"}]}],\"Batch importing: The Job Specification\"]}],[\"$\",\"p\",null,{\"children\":\"The power of this new clause is only fully appreciated if we look at what we had to do before it existed.\"}],[\"$\",\"p\",null,{\"children\":[\"In the \",[\"$\",\"a\",null,{\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"href\":\"https://www.youtube.com/watch?v=1EMBx1XeI9o\",\"children\":\"Batch Import JSON from Amazon S3 into Apache Pinot | StarTree Recipes\"}],\" video (and \",[\"$\",\"a\",null,{\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"href\":\"https://dev.startree.ai/docs/pinot/recipes/ingest-csv-files-from-s3\",\"children\":\"accompanying developer guide\"}],\"), we showed how to ingest data into Pinot from an S3 bucket.\"]}],[\"$\",\"p\",null,{\"children\":\"The contents of that bucket are shown in the screenshot below:\"}],[\"$\",\"p\",null,{\"children\":[\"$\",\"img\",null,{\"alt\":\"Sample data ingested into Apache Pinot from a S3 bucket\",\"src\":\"https://www.datocms-assets.com/75153/1668701275-image4.png\",\"title\":\"Sample data ingested into Apache Pinot from a S3 bucket\"}]}],[\"$\",\"p\",null,{\"children\":\"Let’s quickly recap the steps that we had to do to import those files into Pinot. We have a table called events, which has the following schema:\"}],[\"$\",\"p\",null,{\"children\":[\"$\",\"img\",null,{\"alt\":\"Events schema table\",\"src\":\"https://www.datocms-assets.com/75153/1668701353-image1.png\",\"title\":\"Events schema table\"}]}],[\"$\",\"p\",null,{\"children\":\"We first created a job specification file, which contains a description of our import job. The job file is shown below:\"}],[\"$\",\"$L13\",null,{\"className\":\"language-yaml\",\"children\":[\"$\",\"code\",null,{\"className\":\"code-highlight language-yaml\",\"children\":[[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[[\"$\",\"span\",null,{\"className\":\"token atrule key\",\"children\":\"executionFrameworkSpec\"}],[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\":\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[\" \",[\"$\",\"span\",null,{\"className\":\"token atrule key\",\"children\":\"name\"}],[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\":\"}],\" \",[\"$\",\"span\",null,{\"className\":\"token string\",\"children\":\"'standalone'\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[\" \",[\"$\",\"span\",null,{\"className\":\"token atrule key\",\"children\":\"segmentGenerationJobRunnerClassName\"}],[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\":\"}],\" \",[\"$\",\"span\",null,{\"className\":\"token string\",\"children\":\"'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentGenerationJobRunner'\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[\" \",[\"$\",\"span\",null,{\"className\":\"token atrule key\",\"children\":\"segmentTarPushJobRunnerClassName\"}],[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\":\"}],\" \",[\"$\",\"span\",null,{\"className\":\"token string\",\"children\":\"'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentTarPushJobRunner'\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[\" \",[\"$\",\"span\",null,{\"className\":\"token atrule key\",\"children\":\"segmentUriPushJobRunnerClassName\"}],[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\":\"}],\" \",[\"$\",\"span\",null,{\"className\":\"token string\",\"children\":\"'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentUriPushJobRunner'\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[[\"$\",\"span\",null,{\"className\":\"token atrule key\",\"children\":\"jobType\"}],[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\":\"}],\" SegmentCreationAndTarPush\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[[\"$\",\"span\",null,{\"className\":\"token atrule key\",\"children\":\"inputDirURI\"}],[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\":\"}],\" \",[\"$\",\"span\",null,{\"className\":\"token string\",\"children\":\"'s3://marks-st-cloud-bucket/events/'\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[[\"$\",\"span\",null,{\"className\":\"token atrule key\",\"children\":\"includeFileNamePattern\"}],[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\":\"}],\" \",[\"$\",\"span\",null,{\"className\":\"token string\",\"children\":\"'glob:**/*.json'\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[[\"$\",\"span\",null,{\"className\":\"token atrule key\",\"children\":\"outputDirURI\"}],[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\":\"}],\" \",[\"$\",\"span\",null,{\"className\":\"token string\",\"children\":\"'/data'\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[[\"$\",\"span\",null,{\"className\":\"token atrule key\",\"children\":\"overwriteOutput\"}],[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\":\"}],\" \",[\"$\",\"span\",null,{\"className\":\"token boolean important\",\"children\":\"true\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[[\"$\",\"span\",null,{\"className\":\"token atrule key\",\"children\":\"pinotFSSpecs\"}],[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\":\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[\" \",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\"-\"}],\" \",[\"$\",\"span\",null,{\"className\":\"token atrule key\",\"children\":\"scheme\"}],[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\":\"}],\" s3\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[\" \",[\"$\",\"span\",null,{\"className\":\"token atrule key\",\"children\":\"className\"}],[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\":\"}],\" org.apache.pinot.plugin.filesystem.S3PinotFS\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[\" \",[\"$\",\"span\",null,{\"className\":\"token atrule key\",\"children\":\"configs\"}],[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\":\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[\" \",[\"$\",\"span\",null,{\"className\":\"token atrule key\",\"children\":\"region\"}],[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\":\"}],\" \",[\"$\",\"span\",null,{\"className\":\"token string\",\"children\":\"'eu-west-2'\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[\" \",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\"-\"}],\" \",[\"$\",\"span\",null,{\"className\":\"token atrule key\",\"children\":\"scheme\"}],[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\":\"}],\" file\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[\" \",[\"$\",\"span\",null,{\"className\":\"token atrule key\",\"children\":\"className\"}],[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\":\"}],\" org.apache.pinot.spi.filesystem.LocalPinotFS\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[[\"$\",\"span\",null,{\"className\":\"token atrule key\",\"children\":\"recordReaderSpec\"}],[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\":\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[\" \",[\"$\",\"span\",null,{\"className\":\"token atrule key\",\"children\":\"dataFormat\"}],[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\":\"}],\" \",[\"$\",\"span\",null,{\"className\":\"token string\",\"children\":\"'json'\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[\" \",[\"$\",\"span\",null,{\"className\":\"token atrule key\",\"children\":\"className\"}],[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\":\"}],\" \",[\"$\",\"span\",null,{\"className\":\"token string\",\"children\":\"'org.apache.pinot.plugin.inputformat.json.JSONRecordReader'\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[[\"$\",\"span\",null,{\"className\":\"token atrule key\",\"children\":\"tableSpec\"}],[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\":\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[\" \",[\"$\",\"span\",null,{\"className\":\"token atrule key\",\"children\":\"tableName\"}],[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\":\"}],\" \",[\"$\",\"span\",null,{\"className\":\"token string\",\"children\":\"'events'\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[[\"$\",\"span\",null,{\"className\":\"token atrule key\",\"children\":\"pinotClusterSpecs\"}],[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\":\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[\" \",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\"-\"}],\" \",[\"$\",\"span\",null,{\"className\":\"token atrule key\",\"children\":\"controllerURI\"}],[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\":\"}],\" \",[\"$\",\"span\",null,{\"className\":\"token string\",\"children\":\"'http://${PINOT_CONTROLLER}:9000'\"}],\"\\n\"]}]]}]}],[\"$\",\"p\",null,{\"children\":\"At a high level, this file describes a batch import job that will ingest files from the S3 bucket at s3://marks-st-cloud-bucket/events/ where the files match the glob:**/*.json pattern.\"}],[\"$\",\"p\",null,{\"children\":\"We can import the data by running the following command from the terminal:\"}],[\"$\",\"$L13\",null,{\"className\":\"language-bash\",\"children\":[\"$\",\"code\",null,{\"className\":\"code-highlight language-bash\",\"children\":[[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[[\"$\",\"span\",null,{\"className\":\"token function\",\"children\":\"docker\"}],\" run \",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\"\\\\\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[\" \",[\"$\",\"span\",null,{\"className\":\"token variable parameter\",\"children\":\"--network\"}],\" ingest-json-files-s3 \",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\"\\\\\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[\" \",[\"$\",\"span\",null,{\"className\":\"token variable parameter\",\"children\":\"-v\"}],\" \",[\"$\",\"span\",null,{\"className\":\"token constant environment\",\"children\":\"$$PWD\"}],\"/config:/config \",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\"\\\\\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[\" \",[\"$\",\"span\",null,{\"className\":\"token variable parameter\",\"children\":\"-e\"}],\" \",[\"$\",\"span\",null,{\"className\":\"token variable assign-left\",\"children\":\"AWS_ACCESS_KEY_ID\"}],[\"$\",\"span\",null,{\"className\":\"token operator\",\"children\":\"=\"}],\"AKIARCOCT6DWLUB7F77Z \",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\"\\\\\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[\" \",[\"$\",\"span\",null,{\"className\":\"token variable parameter\",\"children\":\"-e\"}],\" \",[\"$\",\"span\",null,{\"className\":\"token variable assign-left\",\"children\":\"AWS_SECRET_ACCESS_KEY\"}],[\"$\",\"span\",null,{\"className\":\"token operator\",\"children\":\"=\"}],\"gfz71RX+Tj4udve43YePCBqMsIeN1PvHXrVFyxJS \",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\"\\\\\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[\" apachepinot/pinot:0.11.0 LaunchDataIngestionJob \",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\"\\\\\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[\" \",[\"$\",\"span\",null,{\"className\":\"token variable parameter\",\"children\":\"-jobSpecFile\"}],\" /config/job-spec.yml \",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\"\\\\\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[\" \",[\"$\",\"span\",null,{\"className\":\"token variable parameter\",\"children\":\"-values\"}],\" \",[\"$\",\"span\",null,{\"className\":\"token variable assign-left\",\"children\":\"PINOT_CONTROLLER\"}],[\"$\",\"span\",null,{\"className\":\"token operator\",\"children\":\"=\"}],\"pinot-controller\\n\"]}]]}]}],[\"$\",\"p\",null,{\"children\":\"And don’t worry, those credentials have already been deleted; I find it easier to understand what values go where if we use real values.\"}],[\"$\",\"p\",null,{\"children\":[\"Once we’ve run this command, if we go to the Pinot UI at \",[\"$\",\"a\",null,{\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"href\":\"http://localhost:9000/\",\"children\":\"http://localhost:9000\"}],\" and click through to the events table from the Query Console menu, we’ll see that the records have been imported, as shown in the screenshot below:\"]}],[\"$\",\"p\",null,{\"children\":[\"$\",\"img\",null,{\"alt\":\"Sample imported records shown in the Apache Pinot Query Console menu\",\"src\":\"https://www.datocms-assets.com/75153/1668701512-image3.png\",\"title\":\"Sample imported records shown in the Apache Pinot Query Console menu\"}]}],[\"$\",\"p\",null,{\"children\":\"This approach works, and we may still prefer to use it when we need fine-grained control over the ingestion parameters, but it is a bit heavyweight for your everyday data import!\"}],[\"$\",\"h2\",null,{\"id\":\"batch-importing-with-sql\",\"children\":[[\"$\",\"a\",null,{\"href\":\"#batch-importing-with-sql\",\"aria-hidden\":\"true\",\"tabIndex\":\"-1\",\"children\":[\"$\",\"span\",null,{\"className\":\"icon icon-link\"}]}],\"Batch Importing with SQL\"]}],[\"$\",\"p\",null,{\"children\":\"Now let’s do the same thing in SQL.\"}],[\"$\",\"p\",null,{\"children\":\"There are some prerequisites to using the SQL approach, so let’s go through those now, so you don’t end up with a bunch of exceptions when you try this out!\"}],[\"$\",\"p\",null,{\"children\":[\"First of all, you must have a \",[\"$\",\"a\",null,{\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"href\":\"https://docs.pinot.apache.org/basics/components/minion\",\"children\":\"Minion\"}],\" in the Pinot cluster, as this is the component that will do the data import.\"]}],[\"$\",\"p\",null,{\"children\":\"You’ll also need to include the following in your table config:\"}],[\"$\",\"$L13\",null,{\"className\":\"language-json\",\"children\":[\"$\",\"code\",null,{\"className\":\"code-highlight language-json\",\"children\":[[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[[\"$\",\"span\",null,{\"className\":\"token property\",\"children\":\"\\\"task\\\"\"}],[\"$\",\"span\",null,{\"className\":\"token operator\",\"children\":\":\"}],\" \",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\"{\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[\" \",[\"$\",\"span\",null,{\"className\":\"token property\",\"children\":\"\\\"taskTypeConfigsMap\\\"\"}],[\"$\",\"span\",null,{\"className\":\"token operator\",\"children\":\":\"}],\" \",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\"{\"}],\" \",[\"$\",\"span\",null,{\"className\":\"token property\",\"children\":\"\\\"SegmentGenerationAndPushTask\\\"\"}],[\"$\",\"span\",null,{\"className\":\"token operator\",\"children\":\":\"}],\" \",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\"{\"}],[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\"}\"}],\" \",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\"}\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\"}\"}],\"\\n\"]}]]}]}],[\"$\",\"p\",null,{\"children\":\"As long as you’ve done those two things, we’re ready to write our import query! A query that imports JSON files from my S3 bucket is shown below:\"}],[\"$\",\"$L13\",null,{\"className\":\"language-sql\",\"children\":[\"$\",\"code\",null,{\"className\":\"code-highlight language-sql\",\"children\":[[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[[\"$\",\"span\",null,{\"className\":\"token keyword\",\"children\":\"INSERT\"}],\" \",[\"$\",\"span\",null,{\"className\":\"token keyword\",\"children\":\"INTO\"}],\" events\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[[\"$\",\"span\",null,{\"className\":\"token keyword\",\"children\":\"FROM\"}],\" \",[\"$\",\"span\",null,{\"className\":\"token keyword\",\"children\":\"FILE\"}],\" \",[\"$\",\"span\",null,{\"className\":\"token string\",\"children\":\"'s3://marks-st-cloud-bucket/events/'\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[[\"$\",\"span\",null,{\"className\":\"token keyword\",\"children\":\"OPTION\"}],[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\"(\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[\" taskName\",[\"$\",\"span\",null,{\"className\":\"token operator\",\"children\":\"=\"}],\"events\",[\"$\",\"span\",null,{\"className\":\"token operator\",\"children\":\"-\"}],\"task\",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\",\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[\" includeFileNamePattern\",[\"$\",\"span\",null,{\"className\":\"token operator\",\"children\":\"=\"}],\"glob:\",[\"$\",\"span\",null,{\"className\":\"token operator\",\"children\":\"*\"}],[\"$\",\"span\",null,{\"className\":\"token operator\",\"children\":\"*\"}],[\"$\",\"span\",null,{\"className\":\"token operator\",\"children\":\"/\"}],[\"$\",\"span\",null,{\"className\":\"token operator\",\"children\":\"*\"}],[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\".\"}],\"json\",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\",\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[\" input\",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\".\"}],\"fs\",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\".\"}],\"className\",[\"$\",\"span\",null,{\"className\":\"token operator\",\"children\":\"=\"}],\"org\",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\".\"}],\"apache\",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\".\"}],\"pinot\",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\".\"}],\"plugin\",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\".\"}],\"filesystem\",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\".\"}],\"S3PinotFS\",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\",\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[\" input\",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\".\"}],\"fs\",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\".\"}],\"prop\",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\".\"}],\"accessKey\",[\"$\",\"span\",null,{\"className\":\"token operator\",\"children\":\"=\"}],\"AKIARCOCT6DWLUB7F77Z\",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\",\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[\" input\",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\".\"}],\"fs\",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\".\"}],\"prop\",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\".\"}],\"secretKey\",[\"$\",\"span\",null,{\"className\":\"token operator\",\"children\":\"=\"}],\"gfz71RX\",[\"$\",\"span\",null,{\"className\":\"token operator\",\"children\":\"+\"}],\"Tj4udve43YePCBqMsIeN1PvHXrVFyxJS\",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\",\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[\" input\",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\".\"}],\"fs\",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\".\"}],\"prop\",[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\".\"}],\"region\",[\"$\",\"span\",null,{\"className\":\"token operator\",\"children\":\"=\"}],\"eu\",[\"$\",\"span\",null,{\"className\":\"token operator\",\"children\":\"-\"}],\"west\",[\"$\",\"span\",null,{\"className\":\"token operator\",\"children\":\"-\"}],[\"$\",\"span\",null,{\"className\":\"token number\",\"children\":\"2\"}],\"\\n\"]}],[\"$\",\"span\",null,{\"className\":\"code-line\",\"children\":[[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\")\"}],[\"$\",\"span\",null,{\"className\":\"token punctuation\",\"children\":\";\"}],\"\\n\"]}]]}]}],[\"$\",\"p\",null,{\"children\":\"If we run this query, we’ll see the following output:\"}],[\"$\",\"p\",null,{\"children\":[\"$\",\"img\",null,{\"alt\":\"Sample events_OFFLINE query result\",\"src\":\"https://www.datocms-assets.com/75153/1668701654-image5.png\",\"title\":\"Sample events_OFFLINE query result\"}]}],[\"$\",\"p\",null,{\"children\":[\"We can check on the state of the ingestion job via the Swagger REST API. If we navigate to \",[\"$\",\"a\",null,{\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"href\":\"http://localhost:9000/help#/Task/getTaskState\",\"children\":\"http://localhost:9000/help#/Task/getTaskState\"}],\", paste Task_SegmentGenerationAndPushTask_events-task as our task name, and then click Execute, we’ll see the following:\"]}],[\"$\",\"p\",null,{\"children\":[\"$\",\"img\",null,{\"alt\":\"Checking the state of an ingestion job screen\",\"src\":\"https://www.datocms-assets.com/75153/1668701727-image2.png\",\"title\":\"Checking the state of an ingestion job screen\"}]}],[\"$\",\"p\",null,{\"children\":\"If we see the state COMPLETED, this means the data has been ingested, which we can check by going back to the Query console and clicking on the events table.\"}],[\"$\",\"h2\",null,{\"id\":\"summary\",\"children\":[[\"$\",\"a\",null,{\"href\":\"#summary\",\"aria-hidden\":\"true\",\"tabIndex\":\"-1\",\"children\":[\"$\",\"span\",null,{\"className\":\"icon icon-link\"}]}],\"Summary\"]}],[\"$\",\"p\",null,{\"children\":\"I have to say that batch ingestion of data into Apache Pinot has always felt a bit clunky, but with this new clause, it’s super easy, and it’s gonna save us all a bunch of time.\"}],[\"$\",\"p\",null,{\"children\":\"Also, anything that means I’m not writing YAML files has got to be a good thing!\"}],[\"$\",\"p\",null,{\"children\":[\"So give it a try and let us know how you get on. If you have any questions about this feature, feel free to join us on \",[\"$\",\"a\",null,{\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"href\":\"https://stree.ai/slack\",\"children\":\"Slack\"}],\", where we’ll be happy to help you out.\"]}]]}]}]}]]}],[\"$\",\"aside\",null,{\"className\":\"mt-10 hidden border-l-2 pl-5 lg:sticky lg:top-1 lg:block lg:h-full\",\"children\":[\"$\",\"section\",null,{\"className\":\"sticky top-0 mb-4 w-[15.375rem]\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex flex-col space-y-1.5 pb-3\",\"children\":[\"$\",\"h3\",null,{\"className\":\"text-sm font-semibold leading-snug text-neutral-500 dark:text-neutral-100\",\"children\":\"Table of Contents\"}]}],[\"$\",\"$L14\",null,{\"chapters\":[{\"value\":\"Batch importing: The Job Specification\",\"url\":\"#batch-importing-the-job-specification\",\"depth\":2},{\"value\":\"Batch Importing with SQL\",\"url\":\"#batch-importing-with-sql\",\"depth\":2},{\"value\":\"Summary\",\"url\":\"#summary\",\"depth\":2}]}]]}]}]]}]}]]}]]\n"])</script><script>self.__next_f.push([1,"6:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"Apache Pinot™ 0.11 - Inserts from SQL | Apache Pinot™\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"Explore the INSERT INTO clause, which makes ingesting batch data into Pinot as easy as writing a SQL query.\"}],[\"$\",\"meta\",\"4\",{\"name\":\"robots\",\"content\":\"index, follow\"}],[\"$\",\"meta\",\"5\",{\"name\":\"googlebot\",\"content\":\"index, follow, max-video-preview:-1, max-image-preview:large, max-snippet:-1\"}],[\"$\",\"link\",\"6\",{\"rel\":\"canonical\",\"href\":\"https://pinot.apache.org/blog/2022/11/17/Apache%20Pinot-Inserts-from-SQL\"}],[\"$\",\"link\",\"7\",{\"rel\":\"alternate\",\"type\":\"application/rss+xml\",\"href\":\"https://pinot.apache.org/feed.xml\"}],[\"$\",\"meta\",\"8\",{\"property\":\"og:title\",\"content\":\"Apache Pinot™ 0.11 - Inserts from SQL\"}],[\"$\",\"meta\",\"9\",{\"property\":\"og:description\",\"content\":\"Explore the INSERT INTO clause, which makes ingesting batch data into Pinot as easy as writing a SQL query.\"}],[\"$\",\"meta\",\"10\",{\"property\":\"og:url\",\"content\":\"https://pinot.apache.org/blog/2022/11/17/Apache%20Pinot-Inserts-from-SQL\"}],[\"$\",\"meta\",\"11\",{\"property\":\"og:site_name\",\"content\":\"Apache Pinot™\"}],[\"$\",\"meta\",\"12\",{\"property\":\"og:locale\",\"content\":\"en_US\"}],[\"$\",\"meta\",\"13\",{\"property\":\"og:image\",\"content\":\"https://pinot.apache.org/static/images/twitter-card.png\"}],[\"$\",\"meta\",\"14\",{\"property\":\"og:type\",\"content\":\"article\"}],[\"$\",\"meta\",\"15\",{\"property\":\"article:published_time\",\"content\":\"2022-11-17T00:00:00.000Z\"}],[\"$\",\"meta\",\"16\",{\"property\":\"article:modified_time\",\"content\":\"2022-11-17T00:00:00.000Z\"}],[\"$\",\"meta\",\"17\",{\"property\":\"article:author\",\"content\":\"Mark Needham\"}],[\"$\",\"meta\",\"18\",{\"name\":\"twitter:card\",\"content\":\"summary_large_image\"}],[\"$\",\"meta\",\"19\",{\"name\":\"twitter:title\",\"content\":\"Apache Pinot™ 0.11 - Inserts from SQL\"}],[\"$\",\"meta\",\"20\",{\"name\":\"twitter:description\",\"content\":\"Explore the INSERT INTO clause, which makes ingesting batch data into Pinot as easy as writing a SQL query.\"}],[\"$\",\"meta\",\"21\",{\"name\":\"twitter:image\",\"content\":\"https://pinot.apache.org/static/images/twitter-card.png\"}],[\"$\",\"meta\",\"22\",{\"name\":\"next-size-adjust\"}]]\n"])</script><script>self.__next_f.push([1,"f:null\n"])</script><script>self.__next_f.push([1,""])</script></body></html>