| <!doctype html> |
| <html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-current plugin-docs plugin-id-default docs-doc-id-ingestion/native-batch" data-has-hydrated="false"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="generator" content="Docusaurus v2.4.3"> |
| <title data-rh="true">JSON-based batch | Apache® Druid</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:image" content="https://druid.apache.org/img/druid_nav.png"><meta data-rh="true" name="twitter:image" content="https://druid.apache.org/img/druid_nav.png"><meta data-rh="true" property="og:url" content="https://druid.apache.org/docs/29.0.0/ingestion/native-batch"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="current"><meta data-rh="true" name="docusaurus_tag" content="docs-default-current"><meta data-rh="true" name="docsearch:version" content="current"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-current"><meta data-rh="true" property="og:title" content="JSON-based batch | Apache® Druid"><meta data-rh="true" name="description" content="<!--"><meta data-rh="true" property="og:description" content="<!--"><link data-rh="true" rel="icon" href="/img/favicon.png"><link data-rh="true" rel="canonical" href="https://druid.apache.org/docs/29.0.0/ingestion/native-batch"><link data-rh="true" rel="alternate" href="https://druid.apache.org/docs/29.0.0/ingestion/native-batch" hreflang="en"><link data-rh="true" rel="alternate" href="https://druid.apache.org/docs/29.0.0/ingestion/native-batch" hreflang="x-default"><link rel="preconnect" href="https://www.google-analytics.com"> |
| <link rel="preconnect" href="https://www.googletagmanager.com"> |
| <script async src="https://www.googletagmanager.com/gtag/js?id=UA-131010415-1"></script> |
| <script>function gtag(){dataLayer.push(arguments)}window.dataLayer=window.dataLayer||[],gtag("js",new Date),gtag("config","UA-131010415-1",{})</script> |
| |
| |
| |
| |
| |
| <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"> |
| <script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><link rel="stylesheet" href="/assets/css/styles.4b88a1d0.css"> |
| <link rel="preload" href="/assets/js/runtime~main.63d85568.js" as="script"> |
| <link rel="preload" href="/assets/js/main.7181d856.js" as="script"> |
| </head> |
| <body class="navigation-with-keyboard"> |
| <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus"> |
| <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><nav aria-label="Main" class="navbar navbar--fixed-top navbar--dark"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/druid_nav.png" alt="Apache® Druid" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/druid_nav.png" alt="Apache® Druid" class="themedImage_ToTc themedImage--dark_i4oU"></div></a></div><div class="navbar__items navbar__items--right"><a class="navbar__item navbar__link" href="/technology">Technology</a><a class="navbar__item navbar__link" href="/use-cases">Use Cases</a><a class="navbar__item navbar__link" href="/druid-powered">Powered By</a><a class="navbar__item navbar__link" href="/docs/29.0.0/design/">Docs</a><a class="navbar__item navbar__link" href="/community/">Community</a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Apache®</a><ul class="dropdown__menu"><li><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Foundation<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://apachecon.com/?ref=druid.apache.org" target="_blank" rel="noopener noreferrer" class="dropdown__link">Events<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/licenses/" target="_blank" rel="noopener noreferrer" class="dropdown__link">License<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Thanks<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/security/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Security<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Sponsorship<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a class="navbar__item navbar__link" href="/downloads/">Download</a><div class="searchBox_ZlJk"><div class="navbar__search"><span aria-label="expand searchbar" role="button" class="search-icon" tabindex="0"></span><input type="search" id="search_input_react" placeholder="Loading..." aria-label="Search" class="navbar__search-input search-bar" disabled=""></div></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/29.0.0/design/">Getting started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/29.0.0/tutorials/tutorial-msq-extern">Tutorials</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/29.0.0/design/architecture">Design</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/docs/29.0.0/ingestion/">Ingestion</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/29.0.0/ingestion/">Overview</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/29.0.0/ingestion/data-formats">Ingestion concepts</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/29.0.0/multi-stage-query/">SQL-based batch</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/29.0.0/development/extensions-core/kafka-ingestion">Streaming</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/docs/29.0.0/ingestion/native-batch">Classic batch</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/docs/29.0.0/ingestion/native-batch">JSON-based batch</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/29.0.0/ingestion/hadoop">Hadoop-based</a></li></ul></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/29.0.0/ingestion/ingestion-spec">Ingestion spec reference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/29.0.0/ingestion/schema-design">Schema design tips</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/29.0.0/ingestion/concurrent-append-replace">Concurrent append and replace</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/29.0.0/ingestion/faq">Troubleshooting FAQ</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/29.0.0/data-management/">Data management</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/29.0.0/querying/sql">Querying</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/29.0.0/api-reference/">API reference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/29.0.0/configuration/">Configuration</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/29.0.0/operations/web-console">Operations</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/29.0.0/development/overview">Development</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/29.0.0/misc/papers-and-talks">Misc</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/29.0.0/release-info/release-notes">Release info</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Ingestion</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Classic batch</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">JSON-based batch</span><meta itemprop="position" content="3"></li></ul></nav><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>JSON-based batch</h1></header><div class="theme-admonition theme-admonition-info alert alert--info admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg></span>info</div><div class="admonitionContent_S0QG"><p> This page describes JSON-based batch ingestion using <a href="/docs/29.0.0/ingestion/ingestion-spec">ingestion specs</a>. For SQL-based batch ingestion using the <a href="/docs/29.0.0/multi-stage-query/"><code>druid-multi-stage-query</code></a> extension, see <a href="/docs/29.0.0/multi-stage-query/">SQL-based ingestion</a>. Refer to the <a href="/docs/29.0.0/ingestion/#batch">ingestion methods</a> table to determine which ingestion method is right for you.</p></div></div><p>Apache Druid supports the following types of JSON-based batch indexing tasks:</p><ul><li>Parallel task indexing (<code>index_parallel</code>) that can run multiple indexing tasks concurrently. Parallel task works well for production ingestion tasks.</li><li>Simple task indexing (<code>index</code>) that run a single indexing task at a time. Simple task indexing is suitable for development and test environments.</li></ul><p>This topic covers the configuration for <code>index_parallel</code> ingestion specs.</p><p>For related information on batch indexing, see:</p><ul><li><a href="/docs/29.0.0/ingestion/#batch">Batch ingestion method comparison table</a> for a comparison of batch ingestion methods.</li><li><a href="/docs/29.0.0/tutorials/tutorial-batch">Tutorial: Loading a file</a> for a tutorial on JSON-based batch ingestion.</li><li><a href="/docs/29.0.0/ingestion/input-sources">Input sources</a> for possible input sources.</li><li><a href="/docs/29.0.0/ingestion/data-formats#input-format">Source input formats</a> for possible input formats.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="submit-an-indexing-task">Submit an indexing task<a href="#submit-an-indexing-task" class="hash-link" aria-label="Direct link to Submit an indexing task" title="Direct link to Submit an indexing task"></a></h2><p>To run either kind of JSON-based batch indexing task, you can:</p><ul><li>Use the <strong>Load Data</strong> UI in the web console to define and submit an ingestion spec.</li><li>Define an ingestion spec in JSON based upon the <a href="#parallel-indexing-example">examples</a> and reference topics for batch indexing. Then POST the ingestion spec to the <a href="/docs/29.0.0/api-reference/tasks-api">Tasks API endpoint</a>, <code>/druid/indexer/v1/task</code>, the Overlord service. Alternatively, you can use the indexing script included with Druid at <code>bin/post-index-task</code>.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="parallel-task-indexing">Parallel task indexing<a href="#parallel-task-indexing" class="hash-link" aria-label="Direct link to Parallel task indexing" title="Direct link to Parallel task indexing"></a></h2><p>The parallel task type <code>index_parallel</code> is a task for multi-threaded batch indexing. Parallel task indexing only relies on Druid resources. It doesn't depend on other external systems like Hadoop.</p><p>The <code>index_parallel</code> task is a supervisor task that orchestrates |
| the whole indexing process. The supervisor task splits the input data and creates worker tasks to process the individual portions of data.</p><p>Druid issues the worker tasks to the Overlord. The Overlord schedules and runs the workers on MiddleManagers or Indexers. After a worker task successfully processes the assigned input portion, it reports the resulting segment list to the Supervisor task.</p><p>The Supervisor task periodically checks the status of worker tasks. If a task fails, the Supervisor retries the task until the number of retries reaches the configured limit. If all worker tasks succeed, it publishes the reported segments at once and finalizes ingestion.</p><p>The detailed behavior of the parallel task is different depending on the <code>partitionsSpec</code>. See <a href="#partitionsspec"><code>partitionsSpec</code></a> for more details.</p><p>Parallel tasks require the following:</p><ul><li>A splittable <a href="#splittable-input-sources"><code>inputSource</code></a> in the <code>ioConfig</code>. For a list of supported splittable input formats, see <a href="#splittable-input-sources">Splittable input sources</a>.</li><li>The <code>maxNumConcurrentSubTasks</code> greater than 1 in the <code>tuningConfig</code>. Otherwise tasks run sequentially. The <code>index_parallel</code> task reads each input file one by one and creates segments by itself.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="supported-compression-formats">Supported compression formats<a href="#supported-compression-formats" class="hash-link" aria-label="Direct link to Supported compression formats" title="Direct link to Supported compression formats"></a></h3><p>JSON-based batch ingestion supports the following compression formats:</p><ul><li><code>bz2</code></li><li><code>gz</code></li><li><code>xz</code></li><li><code>zip</code></li><li><code>sz</code> (Snappy)</li><li><code>zst</code> (ZSTD)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="implementation-considerations">Implementation considerations<a href="#implementation-considerations" class="hash-link" aria-label="Direct link to Implementation considerations" title="Direct link to Implementation considerations"></a></h3><p>This section covers implementation details to consider when you implement parallel task ingestion.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="volume-control-for-worker-tasks">Volume control for worker tasks<a href="#volume-control-for-worker-tasks" class="hash-link" aria-label="Direct link to Volume control for worker tasks" title="Direct link to Volume control for worker tasks"></a></h4><p>You can control the amount of input data each worker task processes using different configurations depending on the phase in parallel ingestion. |
| See <a href="#partitionsspec"><code>partitionsSpec</code></a> for details about how partitioning affects data volume for tasks.</p><p>For the tasks that read data from the <code>inputSource</code>, you can set the <a href="#split-hint-spec">Split hint spec</a> in the <code>tuningConfig</code>. |
| For the task that merge shuffled segments, you can set the <code>totalNumMergeTasks</code> in the <code>tuningConfig</code>.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="number-of-running-tasks">Number of running tasks<a href="#number-of-running-tasks" class="hash-link" aria-label="Direct link to Number of running tasks" title="Direct link to Number of running tasks"></a></h4><p>The <code>maxNumConcurrentSubTasks</code> in the <code>tuningConfig</code> determines the number of concurrent worker tasks that run in parallel. The Supervisor task checks the number of current running worker tasks and creates more if it's smaller than <code>maxNumConcurrentSubTasks</code> regardless of the number of available task slots. This may affect to other ingestion performance. See <a href="#capacity-planning">Capacity planning</a> section for more details.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="replacing-or-appending-data">Replacing or appending data<a href="#replacing-or-appending-data" class="hash-link" aria-label="Direct link to Replacing or appending data" title="Direct link to Replacing or appending data"></a></h4><p>By default, JSON-based batch ingestion replaces all data in the intervals in your <code>granularitySpec</code> for any segment that it writes to. If you want to add to the segment instead, set the <code>appendToExisting</code> flag in the <code>ioConfig</code>. JSON-based batch ingestion only replaces data in segments where it actively adds data. If there are segments in the intervals for your <code>granularitySpec</code> that don't have data from a task, they remain unchanged. If any existing segments partially overlap with the intervals in the <code>granularitySpec</code>, the portion of those segments outside the interval for the new spec remain visible.</p><p>You can also perform concurrent append and replace tasks. For more information, see <a href="/docs/29.0.0/ingestion/concurrent-append-replace">Concurrent append and replace</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="fully-replacing-existing-segments-using-tombstones">Fully replacing existing segments using tombstones<a href="#fully-replacing-existing-segments-using-tombstones" class="hash-link" aria-label="Direct link to Fully replacing existing segments using tombstones" title="Direct link to Fully replacing existing segments using tombstones"></a></h4><div class="theme-admonition theme-admonition-info alert alert--info admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg></span>info</div><div class="admonitionContent_S0QG"><p>This feature is still experimental.</p></div></div><p>You can set <code>dropExisting</code> flag in the <code>ioConfig</code> to true if you want the ingestion task to replace all existing segments that start and end within the intervals for your <code>granularitySpec</code>. This applies whether or not the new data covers all existing segments. <code>dropExisting</code> only applies when <code>appendToExisting</code> is false and the <code>granularitySpec</code> contains an <code>interval</code>.</p><p>The following examples demonstrate when to set the <code>dropExisting</code> property to true in the <code>ioConfig</code>:</p><p>Consider an existing segment with an interval of 2020-01-01 to 2021-01-01 and <code>YEAR</code> <code>segmentGranularity</code>. You want to overwrite the whole interval of 2020-01-01 to 2021-01-01 with new data using the finer segmentGranularity of <code>MONTH</code>. If the replacement data does not have a record within every months from 2020-01-01 to 2021-01-01 Druid cannot drop the original <code>YEAR</code> segment even if it does include all the replacement data. Set <code>dropExisting</code> to true in this case to replace the original segment at <code>YEAR</code> <code>segmentGranularity</code> since you no longer need it.</p><p>Imagine you want to re-ingest or overwrite a datasource and the new data does not contain some time intervals that exist in the datasource. For example, a datasource contains the following data at <code>MONTH</code> <code>segmentGranularity</code>:</p><ul><li><strong>January</strong>: 1 record </li><li><strong>February</strong>: 10 records </li><li><strong>March</strong>: 10 records </li></ul><p>You want to re-ingest and overwrite with new data as follows:</p><ul><li><strong>January</strong>: 0 records </li><li><strong>February</strong>: 10 records </li><li><strong>March</strong>: 9 records </li></ul><p>Unless you set <code>dropExisting</code> to true, the result after ingestion with overwrite using the same <code>MONTH</code> <code>segmentGranularity</code> would be:</p><ul><li><strong>January</strong>: 1 record </li><li><strong>February</strong>: 10 records </li><li><strong>March</strong>: 9 records</li></ul><p>This may not be what it is expected since the new data has 0 records for January. Set <code>dropExisting</code> to true to replace the unneeded January segment with a tombstone.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="parallel-indexing-example">Parallel indexing example<a href="#parallel-indexing-example" class="hash-link" aria-label="Direct link to Parallel indexing example" title="Direct link to Parallel indexing example"></a></h2><p>The following example illustrates the configuration for a parallel indexing task.</p><details class="details_lb9f alert alert--info details_b_Ee" data-collapsed="true"><summary>Click to view the example</summary><div><div class="collapsibleContent_i85q"><div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"index_parallel"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"spec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"dataSchema"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"dataSource"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"wikipedia_parallel_index_test"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"timestampSpec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"column"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"timestamp"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"dimensionsSpec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"dimensions"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"country"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"page"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"language"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"user"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"unpatrolled"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"newPage"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"robot"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"anonymous"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"namespace"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"continent"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"region"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"city"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"metricsSpec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"count"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"name"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"count"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"doubleSum"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"name"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"added"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"fieldName"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"added"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"doubleSum"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"name"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"deleted"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"fieldName"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"deleted"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"doubleSum"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"name"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"delta"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"fieldName"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"delta"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"granularitySpec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"segmentGranularity"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"DAY"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"queryGranularity"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"second"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"intervals"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"2013-08-31/2013-09-02"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"ioConfig"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"index_parallel"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"inputSource"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"local"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"baseDir"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"examples/indexing/"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"filter"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"wikipedia_index_data*"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"inputFormat"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"json"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"tuningConfig"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"index_parallel"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"partitionsSpec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"single_dim"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"partitionDimension"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"country"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"targetRowsPerSegment"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">5000000</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"maxNumConcurrentSubTasks"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">2</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></details><h2 class="anchor anchorWithStickyNavbar_LWe7" id="parallel-indexing-configuration">Parallel indexing configuration<a href="#parallel-indexing-configuration" class="hash-link" aria-label="Direct link to Parallel indexing configuration" title="Direct link to Parallel indexing configuration"></a></h2><p>The following table defines the primary sections of the input spec:</p><table><thead><tr><th>Property</th><th>Description</th><th>Required</th></tr></thead><tbody><tr><td><code>type</code></td><td>The task type. For parallel task indexing, set the value to <code>index_parallel</code>.</td><td>yes</td></tr><tr><td><code>id</code></td><td>The task ID. If omitted, Druid generates the task ID using the task type, data source name, interval, and date-time stamp.</td><td>no</td></tr><tr><td><code>spec</code></td><td>The ingestion spec that defines the <a href="#dataschema">data schema</a>, <a href="#ioconfig">IO config</a>, and <a href="#tuningconfig">tuning config</a>.</td><td>yes</td></tr><tr><td><code>context</code></td><td>Context to specify various task configuration parameters. See <a href="/docs/29.0.0/ingestion/tasks#context-parameters">Task context parameters</a> for more details.</td><td>no</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dataschema"><code>dataSchema</code><a href="#dataschema" class="hash-link" aria-label="Direct link to dataschema" title="Direct link to dataschema"></a></h3><p>This field is required. In general, it defines the way that Druid stores your data: the primary timestamp column, the dimensions, metrics, and any transformations. For an overview, see <a href="/docs/29.0.0/ingestion/ingestion-spec#dataschema">Ingestion Spec DataSchema</a>.</p><p>When defining the <code>granularitySpec</code> for index parallel, consider the defining <code>intervals</code> explicitly if you know the time range of the data. This way locking failure happens faster and Druid won't accidentally replace data outside the interval range some rows contain unexpected timestamps. The reasoning is as follows:</p><ul><li>If you explicitly define <code>intervals</code>, JSON-based batch ingestion locks all intervals specified when it starts up. Problems with locking become evident quickly when multiple ingestion or indexing tasks try to obtain a lock on the same interval. For example, if a Kafka ingestion task tries to obtain a lock on a locked interval causing the ingestion task fail. Furthermore, if there are rows outside the specified intervals, Druid drops them, avoiding conflict with unexpected intervals.</li><li>If you don't define <code>intervals</code>, JSON-based batch ingestion locks each interval when the interval is discovered. In this case, if the task overlaps with a higher-priority task, issues with conflicting locks occur later in the ingestion process. If the source data includes rows with unexpected timestamps, they may caused unexpected locking of intervals.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="ioconfig"><code>ioConfig</code><a href="#ioconfig" class="hash-link" aria-label="Direct link to ioconfig" title="Direct link to ioconfig"></a></h3><p>The following table lists the properties of a <code>ioConfig</code> object:</p><table><thead><tr><th>Property</th><th>Description</th><th>Default</th><th>Required</th></tr></thead><tbody><tr><td><code>type</code></td><td>The task type. Set to the value to <code>index_parallel</code>.</td><td>none</td><td>yes</td></tr><tr><td><code>inputFormat</code></td><td><a href="/docs/29.0.0/ingestion/data-formats#input-format"><code>inputFormat</code></a> to specify how to parse input data.</td><td>none</td><td>yes</td></tr><tr><td><code>appendToExisting</code></td><td>Creates segments as additional shards of the latest version, effectively appending to the segment set instead of replacing it. This means that you can append new segments to any datasource regardless of its original partitioning scheme. You must use the <code>dynamic</code> partitioning type for the appended segments. If you specify a different partitioning type, the task fails with an error.</td><td>false</td><td>no</td></tr><tr><td><code>dropExisting</code></td><td>If <code>true</code> and <code>appendToExisting</code> is <code>false</code> and the <code>granularitySpec</code> contains an<code>interval</code>, then the ingestion task replaces all existing segments fully contained by the specified <code>interval</code> when the task publishes new segments. If ingestion fails, Druid doesn't change any existing segments. In the case of misconfiguration where either <code>appendToExisting</code> is <code>true</code> or <code>interval</code> isn't specified in <code>granularitySpec</code>, Druid doesn't replace any segments even if <code>dropExisting</code> is <code>true</code>. WARNING: this feature is still experimental.</td><td>false</td><td>no</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tuningconfig"><code>tuningConfig</code><a href="#tuningconfig" class="hash-link" aria-label="Direct link to tuningconfig" title="Direct link to tuningconfig"></a></h3><p>The <code>tuningConfig</code> is optional. Druid uses default parameters if <code>tuningConfig</code> is not specified.</p><p>The following table lists the properties of a <code>tuningConfig</code> object:</p><table><thead><tr><th>Property</th><th>Description</th><th>Default</th><th>Required</th></tr></thead><tbody><tr><td><code>type</code></td><td>The task type. Set the value to<code>index_parallel</code>.</td><td>none</td><td>yes</td></tr><tr><td><code>maxRowsInMemory</code></td><td>Determines when Druid should perform intermediate persists to disk. Normally you don't need to set this. Depending on the nature of your data, if rows are short in terms of bytes. For example, you may not want to store a million rows in memory. In this case, set this value.</td><td>1000000</td><td>no</td></tr><tr><td><code>maxBytesInMemory</code></td><td>Use to determine when Druid should perform intermediate persists to disk. Normally Druid computes this internally and you don't need to set it. This value represents number of bytes to aggregate in heap memory before persisting. This is based on a rough estimate of memory usage and not actual usage. The maximum heap memory usage for indexing is <code>maxBytesInMemory * (2 + maxPendingPersists)</code>. Note that <code>maxBytesInMemory</code> also includes heap usage of artifacts created from intermediary persists. This means that after every persist, the amount of <code>maxBytesInMemory</code> until next persist will decrease. Tasks fail when the sum of bytes of all intermediary persisted artifacts exceeds <code>maxBytesInMemory</code>.</td><td>1/6 of max JVM memory</td><td>no</td></tr><tr><td><code>maxColumnsToMerge</code></td><td>Limit of the number of segments to merge in a single phase when merging segments for publishing. This limit affects the total number of columns present in a set of segments to merge. If the limit is exceeded, segment merging occurs in multiple phases. Druid merges at least 2 segments per phase, regardless of this setting.</td><td>-1 (unlimited)</td><td>no</td></tr><tr><td><code>maxTotalRows</code></td><td>Deprecated. Use <code>partitionsSpec</code> instead. Total number of rows in segments waiting to be pushed. Used to determine when intermediate pushing should occur.</td><td>20000000</td><td>no</td></tr><tr><td><code>numShards</code></td><td>Deprecated. Use <code>partitionsSpec</code> instead. Directly specify the number of shards to create when using a <code>hashed</code> <code>partitionsSpec</code>. If this is specified and <code>intervals</code> is specified in the <code>granularitySpec</code>, the index task can skip the determine intervals/partitions pass through the data.</td><td>null</td><td>no</td></tr><tr><td><code>splitHintSpec</code></td><td>Hint to control the amount of data that each first phase task reads. Druid may ignore the hint depending on the implementation of the input source. See <a href="#split-hint-spec">Split hint spec</a> for more details.</td><td>size-based split hint spec</td><td>no</td></tr><tr><td><code>partitionsSpec</code></td><td>Defines how to partition data in each timeChunk, see <a href="#partitionsspec">PartitionsSpec</a>.</td><td><code>dynamic</code> if <code>forceGuaranteedRollup</code> = false, <code>hashed</code> or <code>single_dim</code> if <code>forceGuaranteedRollup</code> = true</td><td>no</td></tr><tr><td><code>indexSpec</code></td><td>Defines segment storage format options to be used at indexing time, see <a href="/docs/29.0.0/ingestion/ingestion-spec#indexspec">IndexSpec</a>.</td><td>null</td><td>no</td></tr><tr><td><code>indexSpecForIntermediatePersists</code></td><td>Defines segment storage format options to use at indexing time for intermediate persisted temporary segments. You can use this configuration to disable dimension/metric compression on intermediate segments to reduce memory required for final merging. However, if you disable compression on intermediate segments, page cache use my increase while intermediate segments are used before Druid merges them to the final published segment published. See <a href="/docs/29.0.0/ingestion/ingestion-spec#indexspec">IndexSpec</a> for possible values.</td><td>same as <code>indexSpec</code></td><td>no</td></tr><tr><td><code>maxPendingPersists</code></td><td>Maximum number of pending persists that remain not started. If a new intermediate persist exceeds this limit, ingestion blocks until the currently-running persist finishes. Maximum heap memory usage for indexing scales with <code>maxRowsInMemory * (2 + maxPendingPersists)</code>.</td><td>0 (meaning one persist can be running concurrently with ingestion, and none can be queued up)</td><td>no</td></tr><tr><td><code>forceGuaranteedRollup</code></td><td>Forces <a href="/docs/29.0.0/ingestion/rollup">perfect rollup</a>. The perfect rollup optimizes the total size of generated segments and querying time but increases indexing time. If true, specify <code>intervals</code> in the <code>granularitySpec</code> and use either <code>hashed</code> or <code>single_dim</code> for the <code>partitionsSpec</code>. You cannot use this flag in conjunction with <code>appendToExisting</code> of <code>IOConfig</code>. For more details, see <a href="#segment-pushing-modes">Segment pushing modes</a>.</td><td>false</td><td>no</td></tr><tr><td><code>reportParseExceptions</code></td><td>If true, Druid throws exceptions encountered during parsing and halts ingestion. If false, Druid skips unparseable rows and fields.</td><td>false</td><td>no</td></tr><tr><td><code>pushTimeout</code></td><td>Milliseconds to wait to push segments. Must be >= 0, where 0 means to wait forever.</td><td>0</td><td>no</td></tr><tr><td><code>segmentWriteOutMediumFactory</code></td><td>Segment write-out medium to use when creating segments. See <a href="#segmentwriteoutmediumfactory">SegmentWriteOutMediumFactory</a>.</td><td>If not specified, uses the value from <code>druid.peon.defaultSegmentWriteOutMediumFactory.type</code></td><td>no</td></tr><tr><td><code>maxNumConcurrentSubTasks</code></td><td>Maximum number of worker tasks that can be run in parallel at the same time. The supervisor task spawns worker tasks up to <code>maxNumConcurrentSubTasks</code> regardless of the current available task slots. If this value is 1, the supervisor task processes data ingestion on its own instead of spawning worker tasks. If this value is set to too large, the supervisor may create too many worker tasks that block other ingestion tasks. See <a href="#capacity-planning">Capacity planning</a> for more details.</td><td>1</td><td>no</td></tr><tr><td><code>maxRetry</code></td><td>Maximum number of retries on task failures.</td><td>3</td><td>no</td></tr><tr><td><code>maxNumSegmentsToMerge</code></td><td>Max limit for the number of segments that a single task can merge at the same time in the second phase. Used only when <code>forceGuaranteedRollup</code> is true.</td><td>100</td><td>no</td></tr><tr><td><code>totalNumMergeTasks</code></td><td>Total number of tasks that merge segments in the merge phase when <code>partitionsSpec</code> is set to <code>hashed</code> or <code>single_dim</code>.</td><td>10</td><td>no</td></tr><tr><td><code>taskStatusCheckPeriodMs</code></td><td>Polling period in milliseconds to check running task statuses.</td><td>1000</td><td>no</td></tr><tr><td><code>chatHandlerTimeout</code></td><td>Timeout for reporting the pushed segments in worker tasks.</td><td>PT10S</td><td>no</td></tr><tr><td><code>chatHandlerNumRetries</code></td><td>Retries for reporting the pushed segments in worker tasks.</td><td>5</td><td>no</td></tr><tr><td><code>awaitSegmentAvailabilityTimeoutMillis</code></td><td>Milliseconds to wait for the newly indexed segments to become available for query after ingestion completes. If <code><= 0</code>, no wait occurs. If <code>> 0</code>, the task waits for the Coordinator to indicate that the new segments are available for querying. If the timeout expires, the task exits as successful, but the segments are not confirmed as available for query.</td><td>Long</td><td>no (default = 0)</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="split-hint-spec">Split Hint Spec<a href="#split-hint-spec" class="hash-link" aria-label="Direct link to Split Hint Spec" title="Direct link to Split Hint Spec"></a></h3><p>The split hint spec is used to help the supervisor task divide input sources. Each worker task processes a single input division. You can control the amount of data each worker task reads during the first phase.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="size-based-split-hint-spec">Size-based Split Hint Spec<a href="#size-based-split-hint-spec" class="hash-link" aria-label="Direct link to Size-based Split Hint Spec" title="Direct link to Size-based Split Hint Spec"></a></h4><p>The size-based split hint spec affects all splittable input sources except for the HTTP input source and SQL input source.</p><table><thead><tr><th>Property</th><th>Description</th><th>Default</th><th>Required</th></tr></thead><tbody><tr><td><code>type</code></td><td>Set the value to <code>maxSize</code>.</td><td>none</td><td>yes</td></tr><tr><td><code>maxSplitSize</code></td><td>Maximum number of bytes of input files to process in a single subtask. If a single file is larger than the limit, Druid processes the file alone in a single subtask. Druid does not split files across tasks. One subtask will not process more files than <code>maxNumFiles</code> even when their total size is smaller than <code>maxSplitSize</code>. <a href="/docs/29.0.0/configuration/human-readable-byte">Human-readable format</a> is supported.</td><td>1GiB</td><td>no</td></tr><tr><td><code>maxNumFiles</code></td><td>Maximum number of input files to process in a single subtask. This limit avoids task failures when the ingestion spec is too long. There are two known limits on the max size of serialized ingestion spec: the max ZNode size in ZooKeeper (<code>jute.maxbuffer</code>) and the max packet size in MySQL (<code>max_allowed_packet</code>). These limits can cause ingestion tasks fail if the serialized ingestion spec size hits one of them. One subtask will not process more data than <code>maxSplitSize</code> even when the total number of files is smaller than <code>maxNumFiles</code>.</td><td>1000</td><td>no</td></tr></tbody></table><h4 class="anchor anchorWithStickyNavbar_LWe7" id="segments-split-hint-spec">Segments Split Hint Spec<a href="#segments-split-hint-spec" class="hash-link" aria-label="Direct link to Segments Split Hint Spec" title="Direct link to Segments Split Hint Spec"></a></h4><p>The segments split hint spec is used only for <a href="/docs/29.0.0/ingestion/input-sources"><code>DruidInputSource</code></a>.</p><table><thead><tr><th>Property</th><th>Description</th><th>Default</th><th>Required</th></tr></thead><tbody><tr><td><code>type</code></td><td>Set the value to <code>segments</code>.</td><td>none</td><td>yes</td></tr><tr><td><code>maxInputSegmentBytesPerTask</code></td><td>Maximum number of bytes of input segments to process in a single subtask. If a single segment is larger than this number, Druid processes the segment alone in a single subtask. Druid never splits input segments across tasks. A single subtask will not process more segments than <code>maxNumSegments</code> even when their total size is smaller than <code>maxInputSegmentBytesPerTask</code>. <a href="/docs/29.0.0/configuration/human-readable-byte">Human-readable format</a> is supported.</td><td>1GiB</td><td>no</td></tr><tr><td><code>maxNumSegments</code></td><td>Maximum number of input segments to process in a single subtask. This limit avoids failures due to the the ingestion spec being too long. There are two known limits on the max size of serialized ingestion spec: the max ZNode size in ZooKeeper (<code>jute.maxbuffer</code>) and the max packet size in MySQL (<code>max_allowed_packet</code>). These limits can make ingestion tasks fail when the serialized ingestion spec size hits one of them. A single subtask will not process more data than <code>maxInputSegmentBytesPerTask</code> even when the total number of segments is smaller than <code>maxNumSegments</code>.</td><td>1000</td><td>no</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="partitionsspec"><code>partitionsSpec</code><a href="#partitionsspec" class="hash-link" aria-label="Direct link to partitionsspec" title="Direct link to partitionsspec"></a></h3><p>The primary partition for Druid is time. You can define a secondary partitioning method in the partitions spec. Use the <code>partitionsSpec</code> type that applies for your <a href="/docs/29.0.0/ingestion/rollup">rollup</a> method. </p><p>For perfect rollup, you can use:</p><ul><li><code>hashed</code> partitioning based on the hash value of specified dimensions for each row</li><li><code>single_dim</code> based on ranges of values for a single dimension</li><li><code>range</code> based on ranges of values of multiple dimensions.</li></ul><p>For best-effort rollup, use <code>dynamic</code>.</p><p>For an overview, see <a href="/docs/29.0.0/ingestion/partitioning">Partitioning</a>.</p><p>The <code>partitionsSpec</code> types have different characteristics.</p><table><thead><tr><th>PartitionsSpec</th><th>Ingestion speed</th><th>Partitioning method</th><th>Supported rollup mode</th><th>Secondary partition pruning at query time</th></tr></thead><tbody><tr><td><code>dynamic</code></td><td>Fastest</td><td><a href="#dynamic-partitioning">Dynamic partitioning</a> based on the number of rows in a segment.</td><td>Best-effort rollup</td><td>N/A</td></tr><tr><td><code>hashed</code></td><td>Moderate</td><td>Multiple dimension <a href="#hash-based-partitioning">hash-based partitioning</a> may reduce both your datasource size and query latency by improving data locality. See <a href="/docs/29.0.0/ingestion/partitioning">Partitioning</a> for more details.</td><td>Perfect rollup</td><td>The broker can use the partition information to prune segments early to speed up queries. Since the broker knows how to hash <code>partitionDimensions</code> values to locate a segment, given a query including a filter on all the <code>partitionDimensions</code>, the broker can pick up only the segments holding the rows satisfying the filter on <code>partitionDimensions</code> for query processing.<br><br>Note that <code>partitionDimensions</code> must be set at ingestion time to enable secondary partition pruning at query time.</td></tr><tr><td><code>single_dim</code></td><td>Slower</td><td>Single dimension <a href="#single-dimension-range-partitioning">range partitioning</a> may reduce your datasource size and query latency by improving data locality. See <a href="/docs/29.0.0/ingestion/partitioning">Partitioning</a> for more details.</td><td>Perfect rollup</td><td>The broker can use the partition information to prune segments early to speed up queries. Since the broker knows the range of <code>partitionDimension</code> values in each segment, given a query including a filter on the <code>partitionDimension</code>, the broker can pick up only the segments holding the rows satisfying the filter on <code>partitionDimension</code> for query processing.</td></tr><tr><td><code>range</code></td><td>Slowest</td><td>Multiple dimension <a href="#multi-dimension-range-partitioning">range partitioning</a> may reduce your datasource size and query latency by improving data locality. See <a href="/docs/29.0.0/ingestion/partitioning">Partitioning</a> for more details.</td><td>Perfect rollup</td><td>The broker can use the partition information to prune segments early to speed up queries. Since the broker knows the range of <code>partitionDimensions</code> values within each segment, given a query including a filter on the first of the <code>partitionDimensions</code>, the broker can pick up only the segments holding the rows satisfying the filter on the first partition dimension for query processing.</td></tr></tbody></table><h4 class="anchor anchorWithStickyNavbar_LWe7" id="dynamic-partitioning">Dynamic partitioning<a href="#dynamic-partitioning" class="hash-link" aria-label="Direct link to Dynamic partitioning" title="Direct link to Dynamic partitioning"></a></h4><table><thead><tr><th>Property</th><th>Description</th><th>Default</th><th>Required</th></tr></thead><tbody><tr><td><code>type</code></td><td>Set the value to <code>dynamic</code>.</td><td>none</td><td>yes</td></tr><tr><td><code>maxRowsPerSegment</code></td><td>Used in sharding. Determines how many rows are in each segment.</td><td>5000000</td><td>no</td></tr><tr><td><code>maxTotalRows</code></td><td>Total number of rows across all segments waiting for being pushed. Used in determining when intermediate segment push should occur.</td><td>20000000</td><td>no</td></tr></tbody></table><p>With the dynamic partitioning, the parallel index task runs in a single phase spawning multiple worker tasks (type <code>single_phase_sub_task</code>), each of which creates segments.</p><p>How the worker task creates segments:</p><ul><li>Whenever the number of rows in the current segment exceeds |
| <code>maxRowsPerSegment</code>.</li><li>When the total number of rows in all segments across all time chunks reaches to <code>maxTotalRows</code>. At this point the task pushes all segments created so far to the deep storage and creates new ones.</li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="hash-based-partitioning">Hash-based partitioning<a href="#hash-based-partitioning" class="hash-link" aria-label="Direct link to Hash-based partitioning" title="Direct link to Hash-based partitioning"></a></h4><table><thead><tr><th>Property</th><th>Description</th><th>Default</th><th>Required</th></tr></thead><tbody><tr><td><code>type</code></td><td>Set the value to <code>hashed</code>.</td><td>none</td><td>yes</td></tr><tr><td><code>numShards</code></td><td>Directly specify the number of shards to create. If this is specified and <code>intervals</code> is specified in the <code>granularitySpec</code>, the index task can skip the determine intervals/partitions pass through the data. This property and <code>targetRowsPerSegment</code> cannot both be set.</td><td>none</td><td>no</td></tr><tr><td><code>targetRowsPerSegment</code></td><td>A target row count for each partition. If <code>numShards</code> is left unspecified, the Parallel task will determine a partition count automatically such that each partition has a row count close to the target, assuming evenly distributed keys in the input data. A target per-segment row count of 5 million is used if both <code>numShards</code> and <code>targetRowsPerSegment</code> are null.</td><td>null (or 5,000,000 if both <code>numShards</code> and <code>targetRowsPerSegment</code> are null)</td><td>no</td></tr><tr><td><code>partitionDimensions</code></td><td>The dimensions to partition on. Leave blank to select all dimensions.</td><td>null</td><td>no</td></tr><tr><td><code>partitionFunction</code></td><td>A function to compute hash of partition dimensions. See <a href="#hash-partition-function">Hash partition function</a></td><td><code>murmur3_32_abs</code></td><td>no</td></tr></tbody></table><p>The Parallel task with hash-based partitioning is similar to <a href="https://en.wikipedia.org/wiki/MapReduce" target="_blank" rel="noopener noreferrer">MapReduce</a>. |
| The task runs in up to three phases: <code>partial dimension cardinality</code>, <code>partial segment generation</code> and <code>partial segment merge</code>.</p><p>The <code>partial dimension cardinality</code> phase is an optional phase that only runs if <code>numShards</code> is not specified. |
| The Parallel task splits the input data and assigns them to worker tasks based on the split hint spec. |
| Each worker task (type <code>partial_dimension_cardinality</code>) gathers estimates of partitioning dimensions cardinality for |
| each time chunk. The Parallel task will aggregate these estimates from the worker tasks and determine the highest |
| cardinality across all of the time chunks in the input data, dividing this cardinality by <code>targetRowsPerSegment</code> to |
| automatically determine <code>numShards</code>.</p><p>In the <code>partial segment generation</code> phase, just like the Map phase in MapReduce, |
| the Parallel task splits the input data based on the split hint spec |
| and assigns each split to a worker task. Each worker task (type <code>partial_index_generate</code>) reads the assigned split, and partitions rows by the time chunk from <code>segmentGranularity</code> (primary partition key) in the <code>granularitySpec</code> |
| and then by the hash value of <code>partitionDimensions</code> (secondary partition key) in the <code>partitionsSpec</code>. |
| The partitioned data is stored in local storage of |
| the <a href="/docs/29.0.0/design/middlemanager">middleManager</a> or the <a href="/docs/29.0.0/design/indexer">indexer</a>.</p><p>The <code>partial segment merge</code> phase is similar to the Reduce phase in MapReduce. |
| The Parallel task spawns a new set of worker tasks (type <code>partial_index_generic_merge</code>) to merge the partitioned data created in the previous phase. Here, the partitioned data is shuffled based on |
| the time chunk and the hash value of <code>partitionDimensions</code> to be merged; each worker task reads the data falling in the same time chunk and the same hash value from multiple MiddleManager/Indexer processes and merges them to create the final segments. Finally, they push the final segments to the deep storage at once.</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="hash-partition-function">Hash partition function<a href="#hash-partition-function" class="hash-link" aria-label="Direct link to Hash partition function" title="Direct link to Hash partition function"></a></h5><p>In hash partitioning, the partition function is used to compute hash of partition dimensions. The partition dimension values are first serialized into a byte array as a whole, and then the partition function is applied to compute hash of the byte array. Druid currently supports only one partition function.</p><table><thead><tr><th>name</th><th>description</th></tr></thead><tbody><tr><td><code>murmur3_32_abs</code></td><td>Applies an absolute value function to the result of <a href="https://guava.dev/releases/16.0/api/docs/com/google/common/hash/Hashing.html#murmur3_32()" target="_blank" rel="noopener noreferrer"><code>murmur3_32</code></a>.</td></tr></tbody></table><h4 class="anchor anchorWithStickyNavbar_LWe7" id="single-dimension-range-partitioning">Single-dimension range partitioning<a href="#single-dimension-range-partitioning" class="hash-link" aria-label="Direct link to Single-dimension range partitioning" title="Direct link to Single-dimension range partitioning"></a></h4><div class="theme-admonition theme-admonition-info alert alert--info admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg></span>info</div><div class="admonitionContent_S0QG"><p> Single dimension range partitioning is not supported in the sequential mode of the <code>index_parallel</code> task type.</p></div></div><p>Range partitioning has <a href="#benefits-of-range-partitioning">several benefits</a> related to storage footprint and query |
| performance.</p><p>The Parallel task will use one subtask when you set <code>maxNumConcurrentSubTasks</code> to 1.</p><p>When you use this technique to partition your data, segment sizes may be unequally distributed if the data in your <code>partitionDimension</code> is also unequally distributed. Therefore, to avoid imbalance in data layout, review the distribution of values in your source data before deciding on a partitioning strategy.</p><p>Range partitioning is not possible on multi-value dimensions. If the provided |
| <code>partitionDimension</code> is multi-value, your ingestion job will report an error.</p><table><thead><tr><th>Property</th><th>Description</th><th>Default</th><th>Required</th></tr></thead><tbody><tr><td><code>type</code></td><td>Set the value to <code>single_dim</code>.</td><td>none</td><td>yes</td></tr><tr><td><code>partitionDimension</code></td><td>The dimension to partition on. Only rows with a single dimension value are allowed.</td><td>none</td><td>yes</td></tr><tr><td><code>targetRowsPerSegment</code></td><td>Target number of rows to include in a partition, should be a number that targets segments of 500MB<!-- -->~<!-- -->1GB.</td><td>none</td><td>either this or <code>maxRowsPerSegment</code></td></tr><tr><td><code>maxRowsPerSegment</code></td><td>Soft max for the number of rows to include in a partition.</td><td>none</td><td>either this or <code>targetRowsPerSegment</code></td></tr><tr><td><code>assumeGrouped</code></td><td>Assume that input data has already been grouped on time and dimensions. Ingestion will run faster, but may choose sub-optimal partitions if this assumption is violated.</td><td>false</td><td>no</td></tr></tbody></table><p>With <code>single-dim</code> partitioning, the Parallel task runs in 3 phases, |
| i.e., <code>partial dimension distribution</code>, <code>partial segment generation</code>, and <code>partial segment merge</code>. |
| The first phase is to collect some statistics to find |
| the best partitioning and the other 2 phases are to create partial segments |
| and to merge them, respectively, as in hash-based partitioning.</p><p>In the <code>partial dimension distribution</code> phase, the Parallel task splits the input data and |
| assigns them to worker tasks based on the split hint spec. Each worker task (type <code>partial_dimension_distribution</code>) reads |
| the assigned split and builds a histogram for <code>partitionDimension</code>. |
| The Parallel task collects those histograms from worker tasks and finds |
| the best range partitioning based on <code>partitionDimension</code> to evenly |
| distribute rows across partitions. Note that either <code>targetRowsPerSegment</code> |
| or <code>maxRowsPerSegment</code> will be used to find the best partitioning.</p><p>In the <code>partial segment generation</code> phase, the Parallel task spawns new worker tasks (type <code>partial_range_index_generate</code>) |
| to create partitioned data. Each worker task reads a split created as in the previous phase, |
| partitions rows by the time chunk from the <code>segmentGranularity</code> (primary partition key) in the <code>granularitySpec</code> |
| and then by the range partitioning found in the previous phase. |
| The partitioned data is stored in local storage of |
| the <a href="/docs/29.0.0/design/middlemanager">middleManager</a> or the <a href="/docs/29.0.0/design/indexer">indexer</a>.</p><p>In the <code>partial segment merge</code> phase, the parallel index task spawns a new set of worker tasks (type <code>partial_index_generic_merge</code>) to merge the partitioned |
| data created in the previous phase. Here, the partitioned data is shuffled based on |
| the time chunk and the value of <code>partitionDimension</code>; each worker task reads the segments |
| falling in the same partition of the same range from multiple MiddleManager/Indexer processes and merges |
| them to create the final segments. Finally, they push the final segments to the deep storage.</p><div class="theme-admonition theme-admonition-info alert alert--info admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg></span>info</div><div class="admonitionContent_S0QG"><p> Because the task with single-dimension range partitioning makes two passes over the input |
| in <code>partial dimension distribution</code> and <code>partial segment generation</code> phases, |
| the task may fail if the input changes in between the two passes.</p></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="multi-dimension-range-partitioning">Multi-dimension range partitioning<a href="#multi-dimension-range-partitioning" class="hash-link" aria-label="Direct link to Multi-dimension range partitioning" title="Direct link to Multi-dimension range partitioning"></a></h4><div class="theme-admonition theme-admonition-info alert alert--info admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg></span>info</div><div class="admonitionContent_S0QG"><p> Multi-dimension range partitioning is not supported in the sequential mode of the <code>index_parallel</code> task type.</p></div></div><p>Range partitioning has <a href="#benefits-of-range-partitioning">several benefits</a> related to storage footprint and query |
| performance. Multi-dimension range partitioning improves over single-dimension range partitioning by allowing |
| Druid to distribute segment sizes more evenly, and to prune on more dimensions.</p><p>Range partitioning is not possible on multi-value dimensions. If one of the provided |
| <code>partitionDimensions</code> is multi-value, your ingestion job will report an error.</p><table><thead><tr><th>Property</th><th>Description</th><th>Default</th><th>Required</th></tr></thead><tbody><tr><td><code>type</code></td><td>Set the value to <code>range</code>.</td><td>none</td><td>yes</td></tr><tr><td><code>partitionDimensions</code></td><td>An array of dimensions to partition on. Order the dimensions from most frequently queried to least frequently queried. For best results, limit your number of dimensions to between three and five dimensions.</td><td>none</td><td>yes</td></tr><tr><td><code>targetRowsPerSegment</code></td><td>Target number of rows to include in a partition, should be a number that targets segments of 500MB<!-- -->~<!-- -->1GB.</td><td>none</td><td>either this or <code>maxRowsPerSegment</code></td></tr><tr><td>maxRowsPerSegment</td><td>Soft max for the number of rows to include in a partition.</td><td>none</td><td>either this or <code>targetRowsPerSegment</code></td></tr><tr><td><code>assumeGrouped</code></td><td>Assume that input data has already been grouped on time and dimensions. Ingestion will run faster, but may choose sub-optimal partitions if this assumption is violated.</td><td>false</td><td>no</td></tr></tbody></table><h4 class="anchor anchorWithStickyNavbar_LWe7" id="benefits-of-range-partitioning">Benefits of range partitioning<a href="#benefits-of-range-partitioning" class="hash-link" aria-label="Direct link to Benefits of range partitioning" title="Direct link to Benefits of range partitioning"></a></h4><p>Range partitioning, either <code>single_dim</code> or <code>range</code>, has several benefits:</p><ol><li>Lower storage footprint due to combining similar data into the same segments, which improves compressibility.</li><li>Better query performance due to Broker-level segment pruning, which removes segments from |
| consideration when they cannot possibly contain data matching the query filter.</li></ol><p>For Broker-level segment pruning to be effective, you must include partition dimensions in the <code>WHERE</code> clause. Each |
| partition dimension can participate in pruning if the prior partition dimensions (those to its left) are also |
| participating, and if the query uses filters that support pruning.</p><p>Filters that support pruning include:</p><ul><li>Equality on string literals, like <code>x = 'foo'</code> and <code>x IN ('foo', 'bar')</code> where <code>x</code> is a string.</li><li>Comparison between string columns and string literals, like <code>x < 'foo'</code> or other comparisons |
| involving <code><</code>, <code>></code>, <code><=</code>, or <code>>=</code>.</li></ul><p>For example, if you configure the following <code>range</code> partitioning during ingestion:</p><div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token property">"partitionsSpec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"range"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"partitionDimensions"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token string" style="color:rgb(195, 232, 141)">"countryName"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"cityName"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"targetRowsPerSegment"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">5000000</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Then, filters like <code>WHERE countryName = 'United States'</code> or <code>WHERE countryName = 'United States' AND cityName = 'New York'</code> |
| can make use of pruning. However, <code>WHERE cityName = 'New York'</code> cannot make use of pruning, because countryName is not |
| involved. The clause <code>WHERE cityName LIKE 'New%'</code> cannot make use of pruning either, because LIKE filters do not |
| support pruning.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="http-status-endpoints">HTTP status endpoints<a href="#http-status-endpoints" class="hash-link" aria-label="Direct link to HTTP status endpoints" title="Direct link to HTTP status endpoints"></a></h2><p>The Supervisor task provides some HTTP endpoints to get running status.</p><p><code>http://{PEON_IP}:{PEON_PORT}/druid/worker/v1/chat/{SUPERVISOR_TASK_ID}/mode</code><br> |
| <!-- -->Returns <code>parallel</code> if the indexing task is running in parallel. Otherwise, it returns <code>sequential</code>.</p><p><code>http://{PEON_IP}:{PEON_PORT}/druid/worker/v1/chat/{SUPERVISOR_TASK_ID}/phase</code><br> |
| <!-- -->Returns the name of the current phase if the task running in the parallel mode.</p><p><code>http://{PEON_IP}:{PEON_PORT}/druid/worker/v1/chat/{SUPERVISOR_TASK_ID}/progress</code><br> |
| <!-- -->Returns the estimated progress of the current phase if the supervisor task is running in the parallel mode.</p><p>An example of the result is</p><div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"running"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token number" style="color:rgb(247, 140, 108)">10</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"succeeded"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token number" style="color:rgb(247, 140, 108)">0</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"failed"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token number" style="color:rgb(247, 140, 108)">0</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"complete"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token number" style="color:rgb(247, 140, 108)">0</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"total"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token number" style="color:rgb(247, 140, 108)">10</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"estimatedExpectedSucceeded"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token number" style="color:rgb(247, 140, 108)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><code>http://{PEON_IP}:{PEON_PORT}/druid/worker/v1/chat/{SUPERVISOR_TASK_ID}/subtasks/running</code><br> |
| <!-- -->Returns the task IDs of running worker tasks, or an empty list if the supervisor task is running in the sequential mode.</p><p><code>http://{PEON_IP}:{PEON_PORT}/druid/worker/v1/chat/{SUPERVISOR_TASK_ID}/subtaskspecs</code><br> |
| <!-- -->Returns all worker task specs, or an empty list if the supervisor task is running in the sequential mode.</p><p><code>http://{PEON_IP}:{PEON_PORT}/druid/worker/v1/chat/{SUPERVISOR_TASK_ID}/subtaskspecs/running</code><br> |
| <!-- -->Returns running worker task specs, or an empty list if the supervisor task is running in the sequential mode.</p><p><code>http://{PEON_IP}:{PEON_PORT}/druid/worker/v1/chat/{SUPERVISOR_TASK_ID}/subtaskspecs/complete</code><br> |
| <!-- -->Returns complete worker task specs, or an empty list if the supervisor task is running in the sequential mode.</p><p><code>http://{PEON_IP}:{PEON_PORT}/druid/worker/v1/chat/{SUPERVISOR_TASK_ID}/subtaskspec/{SUB_TASK_SPEC_ID}</code><br> |
| <!-- -->Returns the worker task spec of the given id, or HTTP 404 Not Found error if the supervisor task is running in the sequential mode.</p><p><code>http://{PEON_IP}:{PEON_PORT}/druid/worker/v1/chat/{SUPERVISOR_TASK_ID}/subtaskspec/{SUB_TASK_SPEC_ID}/state</code><br> |
| <!-- -->Returns the state of the worker task spec of the given id, or HTTP 404 Not Found error if the supervisor task is running in the sequential mode. |
| The returned result contains the worker task spec, a current task status if exists, and task attempt history.</p><details class="details_lb9f alert alert--info details_b_Ee" data-collapsed="true"><summary>Click to view the response</summary><div><div class="collapsibleContent_i85q"><div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"spec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"id"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"index_parallel_lineitem_2018-04-20T22:12:43.610Z_2"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"groupId"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"index_parallel_lineitem_2018-04-20T22:12:43.610Z"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"supervisorTaskId"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"index_parallel_lineitem_2018-04-20T22:12:43.610Z"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"context"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token null keyword" style="font-style:italic">null</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"inputSplit"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"split"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"/path/to/data/lineitem.tbl.5"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"ingestionSpec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"dataSchema"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"dataSource"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"lineitem"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"timestampSpec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"column"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_shipdate"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"format"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"yyyy-MM-dd"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"dimensionsSpec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"dimensions"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_orderkey"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_partkey"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_suppkey"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_linenumber"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_returnflag"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_linestatus"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_shipdate"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_commitdate"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_receiptdate"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_shipinstruct"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_shipmode"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_comment"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"metricsSpec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"count"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"name"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"count"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"longSum"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"name"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_quantity"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"fieldName"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_quantity"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"expression"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token null keyword" style="font-style:italic">null</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"doubleSum"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"name"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_extendedprice"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"fieldName"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_extendedprice"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"expression"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token null keyword" style="font-style:italic">null</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"doubleSum"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"name"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_discount"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"fieldName"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_discount"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"expression"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token null keyword" style="font-style:italic">null</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"doubleSum"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"name"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_tax"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"fieldName"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_tax"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"expression"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token null keyword" style="font-style:italic">null</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"granularitySpec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"uniform"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"segmentGranularity"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"YEAR"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"queryGranularity"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"none"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"rollup"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token boolean" style="color:rgb(255, 88, 116)">true</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"intervals"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"1980-01-01T00:00:00.000Z/2020-01-01T00:00:00.000Z"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"transformSpec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"filter"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token null keyword" style="font-style:italic">null</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"transforms"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"ioConfig"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"index_parallel"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"inputSource"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"local"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"baseDir"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"/path/to/data/"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"filter"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"lineitem.tbl.5"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"inputFormat"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"tsv"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"delimiter"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"|"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"columns"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_orderkey"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_partkey"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_suppkey"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_linenumber"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_quantity"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_extendedprice"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_discount"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_tax"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_returnflag"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_linestatus"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_shipdate"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_commitdate"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_receiptdate"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_shipinstruct"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_shipmode"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"l_comment"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"appendToExisting"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token boolean" style="color:rgb(255, 88, 116)">false</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"dropExisting"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token boolean" style="color:rgb(255, 88, 116)">false</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"tuningConfig"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"index_parallel"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"partitionsSpec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"dynamic"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"maxRowsInMemory"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">1000000</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"maxTotalRows"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">20000000</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"numShards"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token null keyword" style="font-style:italic">null</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"indexSpec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"bitmap"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"roaring"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"dimensionCompression"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"lz4"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"metricCompression"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"lz4"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"longEncoding"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"longs"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"indexSpecForIntermediatePersists"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"bitmap"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"roaring"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"dimensionCompression"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"lz4"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"metricCompression"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"lz4"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"longEncoding"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"longs"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"maxPendingPersists"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">0</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"reportParseExceptions"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token boolean" style="color:rgb(255, 88, 116)">false</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"pushTimeout"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">0</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"segmentWriteOutMediumFactory"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token null keyword" style="font-style:italic">null</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"maxNumConcurrentSubTasks"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">4</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"maxRetry"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">3</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"taskStatusCheckPeriodMs"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">1000</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"chatHandlerTimeout"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"PT10S"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"chatHandlerNumRetries"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">5</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"logParseExceptions"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token boolean" style="color:rgb(255, 88, 116)">false</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"maxParseExceptions"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">2147483647</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"maxSavedParseExceptions"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">0</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"forceGuaranteedRollup"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token boolean" style="color:rgb(255, 88, 116)">false</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"currentStatus"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"id"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"index_sub_lineitem_2018-04-20T22:16:29.922Z"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"index_sub"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"createdTime"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"2018-04-20T22:16:29.925Z"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"queueInsertionTime"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"2018-04-20T22:16:29.929Z"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"statusCode"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"RUNNING"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"duration"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">-1</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"location"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"host"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token null keyword" style="font-style:italic">null</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"port"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">-1</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"tlsPort"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">-1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"dataSource"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"lineitem"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"errorMsg"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token null keyword" style="font-style:italic">null</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"taskHistory"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></details><p><code>http://{PEON_IP}:{PEON_PORT}/druid/worker/v1/chat/{SUPERVISOR_TASK_ID}/subtaskspec/{SUB_TASK_SPEC_ID}/history</code><br> |
| <!-- -->Returns the task attempt history of the worker task spec of the given id, or HTTP 404 Not Found error if the supervisor task is running in the sequential mode.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="segment-pushing-modes">Segment pushing modes<a href="#segment-pushing-modes" class="hash-link" aria-label="Direct link to Segment pushing modes" title="Direct link to Segment pushing modes"></a></h2><p>While ingesting data using the parallel task indexing, Druid creates segments from the input data and pushes them. For segment pushing, |
| the parallel task index supports the following segment pushing modes based upon your type of <a href="/docs/29.0.0/ingestion/rollup">rollup</a>:</p><ul><li>Bulk pushing mode: Used for perfect rollup. Druid pushes every segment at the very end of the index task. Until then, Druid stores created segments in memory and local storage of the service running the index task. To enable bulk pushing mode, set <code>forceGuaranteedRollup</code> to <code>true</code> in your tuning config. You cannot use bulk pushing with <code>appendToExisting</code> in your IOConfig.</li><li>Incremental pushing mode: Used for best-effort rollup. Druid pushes segments are incrementally during the course of the indexing task. The index task collects data and stores created segments in the memory and disks of the services running the task until the total number of collected rows exceeds <code>maxTotalRows</code>. At that point the index task immediately pushes all segments created up until that moment, cleans up pushed segments, and continues to ingest the remaining data.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="capacity-planning">Capacity planning<a href="#capacity-planning" class="hash-link" aria-label="Direct link to Capacity planning" title="Direct link to Capacity planning"></a></h2><p>The Supervisor task can create up to <code>maxNumConcurrentSubTasks</code> worker tasks no matter how many task slots are currently available. |
| As a result, total number of tasks which can be run at the same time is <code>(maxNumConcurrentSubTasks + 1)</code> (including the Supervisor task). |
| Please note that this can be even larger than total number of task slots (sum of the capacity of all workers). |
| If <code>maxNumConcurrentSubTasks</code> is larger than <code>n (available task slots)</code>, then |
| <code>maxNumConcurrentSubTasks</code> tasks are created by the supervisor task, but only <code>n</code> tasks would be started. |
| Others will wait in the pending state until any running task is finished.</p><p>If you are using the Parallel Index Task with stream ingestion together, |
| we would recommend to limit the max capacity for batch ingestion to prevent |
| stream ingestion from being blocked by batch ingestion. Suppose you have |
| <code>t</code> Parallel Index Tasks to run at the same time, but want to limit |
| the max number of tasks for batch ingestion to <code>b</code>. Then, (sum of <code>maxNumConcurrentSubTasks</code> |
| of all Parallel Index Tasks + <code>t</code> (for supervisor tasks)) must be smaller than <code>b</code>.</p><p>If you have some tasks of a higher priority than others, you may set their |
| <code>maxNumConcurrentSubTasks</code> to a higher value than lower priority tasks. |
| This may help the higher priority tasks to finish earlier than lower priority tasks |
| by assigning more task slots to them.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="splittable-input-sources">Splittable input sources<a href="#splittable-input-sources" class="hash-link" aria-label="Direct link to Splittable input sources" title="Direct link to Splittable input sources"></a></h2><p>Use the <code>inputSource</code> object to define the location where your index can read data. Only the native parallel task and simple task support the input source.</p><p>For details on available input sources see:</p><ul><li><a href="/docs/29.0.0/ingestion/input-sources#s3-input-source">S3 input source</a> (<code>s3</code>) reads data from AWS S3 storage.</li><li><a href="/docs/29.0.0/ingestion/input-sources#google-cloud-storage-input-source">Google Cloud Storage input source</a> (<code>gs</code>) reads data from Google Cloud Storage.</li><li><a href="/docs/29.0.0/ingestion/input-sources#azure-input-source">Azure input source</a> (<code>azure</code>) reads data from Azure Blob Storage and Azure Data Lake.</li><li><a href="/docs/29.0.0/ingestion/input-sources#hdfs-input-source">HDFS input source</a> (<code>hdfs</code>) reads data from HDFS storage.</li><li><a href="/docs/29.0.0/ingestion/input-sources#http-input-source">HTTP input Source</a> (<code>http</code>) reads data from HTTP servers.</li><li><a href="/docs/29.0.0/ingestion/input-sources#inline-input-source">Inline input Source</a> reads data you paste into the web console.</li><li><a href="/docs/29.0.0/ingestion/input-sources#local-input-source">Local input Source</a> (<code>local</code>) reads data from local storage.</li><li><a href="/docs/29.0.0/ingestion/input-sources#druid-input-source">Druid input Source</a> (<code>druid</code>) reads data from a Druid datasource.</li><li><a href="/docs/29.0.0/ingestion/input-sources#sql-input-source">SQL input Source</a> (<code>sql</code>) reads data from a RDBMS source.</li></ul><p>For information on how to combine input sources, see <a href="/docs/29.0.0/ingestion/input-sources#combining-input-source">Combining input source</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="segmentwriteoutmediumfactory"><code>segmentWriteOutMediumFactory</code><a href="#segmentwriteoutmediumfactory" class="hash-link" aria-label="Direct link to segmentwriteoutmediumfactory" title="Direct link to segmentwriteoutmediumfactory"></a></h3><table><thead><tr><th>Property</th><th>Type</th><th>Description</th><th>Required</th></tr></thead><tbody><tr><td><code>type</code></td><td>String</td><td>See <a href="/docs/29.0.0/configuration/#segmentwriteoutmediumfactory">Additional Peon Configuration: SegmentWriteOutMediumFactory</a> for explanation and available options.</td><td>yes</td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/docs/29.0.0/development/extensions-core/kinesis-ingestion"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Amazon Kinesis</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/docs/29.0.0/ingestion/hadoop"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Hadoop-based</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#submit-an-indexing-task" class="table-of-contents__link toc-highlight">Submit an indexing task</a></li><li><a href="#parallel-task-indexing" class="table-of-contents__link toc-highlight">Parallel task indexing</a><ul><li><a href="#supported-compression-formats" class="table-of-contents__link toc-highlight">Supported compression formats</a></li><li><a href="#implementation-considerations" class="table-of-contents__link toc-highlight">Implementation considerations</a></li></ul></li><li><a href="#parallel-indexing-example" class="table-of-contents__link toc-highlight">Parallel indexing example</a></li><li><a href="#parallel-indexing-configuration" class="table-of-contents__link toc-highlight">Parallel indexing configuration</a><ul><li><a href="#dataschema" class="table-of-contents__link toc-highlight"><code>dataSchema</code></a></li><li><a href="#ioconfig" class="table-of-contents__link toc-highlight"><code>ioConfig</code></a></li><li><a href="#tuningconfig" class="table-of-contents__link toc-highlight"><code>tuningConfig</code></a></li><li><a href="#split-hint-spec" class="table-of-contents__link toc-highlight">Split Hint Spec</a></li><li><a href="#partitionsspec" class="table-of-contents__link toc-highlight"><code>partitionsSpec</code></a></li></ul></li><li><a href="#http-status-endpoints" class="table-of-contents__link toc-highlight">HTTP status endpoints</a></li><li><a href="#segment-pushing-modes" class="table-of-contents__link toc-highlight">Segment pushing modes</a></li><li><a href="#capacity-planning" class="table-of-contents__link toc-highlight">Capacity planning</a></li><li><a href="#splittable-input-sources" class="table-of-contents__link toc-highlight">Splittable input sources</a><ul><li><a href="#segmentwriteoutmediumfactory" class="table-of-contents__link toc-highlight"><code>segmentWriteOutMediumFactory</code></a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer"><div class="container container-fluid"><div class="footer__bottom text--center"><div class="margin-bottom--sm"><img src="/img/favicon.png" class="themedImage_ToTc themedImage--light_HNdA footer__logo"><img src="/img/favicon.png" class="themedImage_ToTc themedImage--dark_i4oU footer__logo"></div><div class="footer__copyright">Copyright © 2023 Apache Software Foundation. Except where otherwise noted, licensed under CC BY-SA 4.0. Apache Druid, Druid, and the Druid logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</div></div></div></footer></div> |
| <script src="/assets/js/runtime~main.63d85568.js"></script> |
| <script src="/assets/js/main.7181d856.js"></script> |
| </body> |
| </html> |
