| <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><nav aria-label="Main" class="navbar navbar--fixed-top navbar--dark"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/druid_nav.png" alt="Apache® Druid" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/druid_nav.png" alt="Apache® Druid" class="themedImage_ToTc themedImage--dark_i4oU"></div></a></div><div class="navbar__items navbar__items--right"><a class="navbar__item navbar__link" href="/technology">Technology</a><a class="navbar__item navbar__link" href="/use-cases">Use Cases</a><a class="navbar__item navbar__link" href="/druid-powered">Powered By</a><a class="navbar__item navbar__link" href="/docs/28.0.1/design/">Docs</a><a class="navbar__item navbar__link" href="/community/">Community</a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Apache®</a><ul class="dropdown__menu"><li><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Foundation<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://apachecon.com/?ref=druid.apache.org" target="_blank" rel="noopener noreferrer" class="dropdown__link">Events<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/licenses/" target="_blank" rel="noopener noreferrer" class="dropdown__link">License<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Thanks<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/security/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Security<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Sponsorship<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a class="navbar__item navbar__link" href="/downloads/">Download</a><div class="searchBox_ZlJk"><div class="navbar__search"><span aria-label="expand searchbar" role="button" class="search-icon" tabindex="0"></span><input type="search" id="search_input_react" placeholder="Loading..." aria-label="Search" class="navbar__search-input search-bar" disabled=""></div></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.1/design/">Getting started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.1/tutorials/tutorial-msq-extern">Tutorials</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.1/design/architecture">Design</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.1/ingestion/">Ingestion</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/docs/28.0.1/data-management/">Data management</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/28.0.1/data-management/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/28.0.1/data-management/update">Data updates</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/28.0.1/data-management/delete">Data deletion</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/28.0.1/data-management/schema-changes">Schema changes</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--active" aria-expanded="true" tabindex="0" href="/docs/28.0.1/data-management/compaction">Compaction</a><button aria-label="Toggle the collapsible sidebar category 'Compaction'" type="button" class="clean-btn menu__caret"></button></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/28.0.1/data-management/automatic-compaction">Automatic compaction</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/docs/28.0.1/data-management/manual-compaction">Manual compaction</a></li></ul></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.1/querying/sql">Querying</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.1/api-reference/">API reference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.1/configuration/">Configuration</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.1/operations/web-console">Operations</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.1/development/overview">Development</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.1/misc/papers-and-talks">Misc</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.1/release-info/release-notes">Release info</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Data management</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item"><a class="breadcrumbs__link" itemprop="item" href="/docs/28.0.1/data-management/compaction"><span itemprop="name">Compaction</span></a><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Manual compaction</span><meta itemprop="position" content="3"></li></ul></nav><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Manual compaction</h1></header><p>In Apache Druid, compaction is a special type of ingestion task that reads data from a Druid datasource and writes it back into the same datasource. A common use case for this is to <a href="/docs/28.0.1/operations/segment-optimization">optimally size segments</a> after ingestion to improve query performance.</p><p>You can perform manual compaction where you submit a one-time compaction task for a specific interval. Generally, you don't need to do this if you use <a href="/docs/28.0.1/data-management/automatic-compaction">automatic compaction</a>, which is recommended for most workloads.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="setting-up-manual-compaction">Setting up manual compaction<a href="#setting-up-manual-compaction" class="hash-link" aria-label="Direct link to Setting up manual compaction" title="Direct link to Setting up manual compaction"></a></h2><p> Compaction tasks merge all segments for the defined interval according to the following syntax:</p><div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"compact"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"id"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> <task_id></span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"dataSource"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> <task_datasource></span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"ioConfig"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> <IO config></span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"dimensionsSpec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> <custom dimensionsSpec></span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"transformSpec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> <custom transformSpec></span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"metricsSpec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> <custom metricsSpec></span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"tuningConfig"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> <parallel indexing task tuningConfig></span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"granularitySpec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> <compaction task granularitySpec></span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"context"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> <task context></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><table><thead><tr><th>Field</th><th>Description</th><th>Required</th></tr></thead><tbody><tr><td><code>type</code></td><td>Task type. Set the value to <code>compact</code>.</td><td>Yes</td></tr><tr><td><code>id</code></td><td>Task ID</td><td>No</td></tr><tr><td><code>dataSource</code></td><td>Data source name to compact</td><td>Yes</td></tr><tr><td><code>ioConfig</code></td><td>I/O configuration for compaction task. See <a href="#compaction-io-configuration">Compaction I/O configuration</a> for details.</td><td>Yes</td></tr><tr><td><code>dimensionsSpec</code></td><td>When set, the compaction task uses the specified <code>dimensionsSpec</code> rather than generating one from existing segments. See <a href="#compaction-dimensions-spec">Compaction dimensionsSpec</a> for details.</td><td>No</td></tr><tr><td><code>transformSpec</code></td><td>When set, the compaction task uses the specified <code>transformSpec</code> rather than using <code>null</code>. See <a href="#compaction-transform-spec">Compaction transformSpec</a> for details.</td><td>No</td></tr><tr><td><code>metricsSpec</code></td><td>When set, the compaction task uses the specified <code>metricsSpec</code> rather than generating one from existing segments.</td><td>No</td></tr><tr><td><code>segmentGranularity</code></td><td>Deprecated. Use <code>granularitySpec</code>.</td><td>No</td></tr><tr><td><code>tuningConfig</code></td><td><a href="/docs/28.0.1/ingestion/native-batch#tuningconfig">Tuning configuration</a> for parallel indexing. <code>awaitSegmentAvailabilityTimeoutMillis</code> value is not supported for compaction tasks. Leave this parameter at the default value, 0.</td><td>No</td></tr><tr><td><code>granularitySpec</code></td><td>When set, the compaction task uses the specified <code>granularitySpec</code> rather than generating one from existing segments. See <a href="#compaction-granularity-spec">Compaction <code>granularitySpec</code></a> for details.</td><td>No</td></tr><tr><td><code>context</code></td><td><a href="/docs/28.0.1/ingestion/tasks#context">Task context</a></td><td>No</td></tr></tbody></table><div class="theme-admonition theme-admonition-info alert alert--info admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg></span>info</div><div class="admonitionContent_S0QG"><p> Note: Use <code>granularitySpec</code> over <code>segmentGranularity</code> and only set one of these values. If you specify different values for these in the same compaction spec, the task fails.</p></div></div><p>To control the number of result segments per time chunk, you can set <a href="/docs/28.0.1/ingestion/native-batch#partitionsspec"><code>maxRowsPerSegment</code></a> or <a href="/docs/28.0.1/ingestion/native-batch#tuningconfig"><code>numShards</code></a>.</p><div class="theme-admonition theme-admonition-info alert alert--info admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg></span>info</div><div class="admonitionContent_S0QG"><p> You can run multiple compaction tasks in parallel. For example, if you want to compact the data for a year, you are not limited to running a single task for the entire year. You can run 12 compaction tasks with month-long intervals.</p></div></div><p>A compaction task internally generates an <code>index</code> or <code>index_parallel</code> task spec for performing compaction work with some fixed parameters. For example, its <code>inputSource</code> is always the <a href="/docs/28.0.1/ingestion/input-sources"><code>druid</code> input source</a>, and <code>dimensionsSpec</code> and <code>metricsSpec</code> include all dimensions and metrics of the input segments by default.</p><p>Compaction tasks typically fetch all <a href="#compaction-io-configuration">relevant segments</a> prior to launching any subtasks, <em>unless</em> the following properties are all set to non-null values. It is strongly recommended to set them to non-null values to maximize performance and minimize disk usage of the <code>compact</code> task:</p><ul><li><a href="#compaction-granularity-spec"><code>granularitySpec</code></a>, with non-null values for each of <code>segmentGranularity</code>, <code>queryGranularity</code>, and <code>rollup</code></li><li><a href="#compaction-dimensions-spec"><code>dimensionsSpec</code></a></li><li><code>metricsSpec</code></li></ul><p>Compaction tasks exit without doing anything and issue a failure status code in either of the following cases:</p><ul><li>If the interval you specify has no data segments loaded.</li><li>If the interval you specify is empty.</li></ul><p>Note that the metadata between input segments and the resulting compacted segments may differ if the metadata among the input segments differs as well. If all input segments have the same metadata, however, the resulting output segment will have the same metadata as all input segments.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="manual-compaction-task-example">Manual compaction task example<a href="#manual-compaction-task-example" class="hash-link" aria-label="Direct link to Manual compaction task example" title="Direct link to Manual compaction task example"></a></h2><p>The following JSON illustrates a compaction task to compact <em>all segments</em> within the interval <code>2020-01-01/2021-01-01</code> and create new segments:</p><div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"compact"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"dataSource"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"wikipedia"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"ioConfig"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"compact"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"inputSpec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"interval"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"interval"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"2020-01-01/2021-01-01"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"granularitySpec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"segmentGranularity"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"day"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"queryGranularity"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"hour"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><code>granularitySpec</code> is an optional field. |
| If you don't specify <code>granularitySpec</code>, Druid retains the original segment and query granularities when compaction is complete.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="compaction-io-configuration">Compaction I/O configuration<a href="#compaction-io-configuration" class="hash-link" aria-label="Direct link to Compaction I/O configuration" title="Direct link to Compaction I/O configuration"></a></h2><p>The compaction <code>ioConfig</code> requires specifying <code>inputSpec</code> as follows:</p><table><thead><tr><th>Field</th><th>Description</th><th>Default</th><th>Required</th></tr></thead><tbody><tr><td><code>type</code></td><td>Task type. Set the value to <code>compact</code>.</td><td>none</td><td>Yes</td></tr><tr><td><code>inputSpec</code></td><td>Specification of the target <a href="#interval-inputspec">interval</a> or <a href="#segments-inputspec">segments</a>.</td><td>none</td><td>Yes</td></tr><tr><td><code>dropExisting</code></td><td>If <code>true</code>, the task replaces all existing segments fully contained by either of the following:<br>- the <code>interval</code> in the <code>interval</code> type <code>inputSpec</code>.<br>- the umbrella interval of the <code>segments</code> in the <code>segment</code> type <code>inputSpec</code>.<br>If compaction fails, Druid does not change any of the existing segments.<br><strong>WARNING</strong>: <code>dropExisting</code> in <code>ioConfig</code> is a beta feature.</td><td>false</td><td>No</td></tr><tr><td><code>allowNonAlignedInterval</code></td><td>If <code>true</code>, the task allows an explicit <a href="#compaction-granularity-spec"><code>segmentGranularity</code></a> that is not aligned with the provided <a href="#interval-inputspec">interval</a> or <a href="#segments-inputspec">segments</a>. This parameter is only used if <a href="#compaction-granularity-spec"><code>segmentGranularity</code></a> is explicitly provided.<br><br>This parameter is provided for backwards compatibility. In most scenarios it should not be set, as it can lead to data being accidentally overshadowed. This parameter may be removed in a future release.</td><td>false</td><td>No</td></tr></tbody></table><p>The compaction task has two kinds of <code>inputSpec</code>:</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="interval-inputspec">Interval <code>inputSpec</code><a href="#interval-inputspec" class="hash-link" aria-label="Direct link to interval-inputspec" title="Direct link to interval-inputspec"></a></h3><table><thead><tr><th>Field</th><th>Description</th><th>Required</th></tr></thead><tbody><tr><td><code>type</code></td><td>Task type. Set the value to <code>interval</code>.</td><td>Yes</td></tr><tr><td><code>interval</code></td><td>Interval to compact.</td><td>Yes</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="segments-inputspec">Segments <code>inputSpec</code><a href="#segments-inputspec" class="hash-link" aria-label="Direct link to segments-inputspec" title="Direct link to segments-inputspec"></a></h3><table><thead><tr><th>Field</th><th>Description</th><th>Required</th></tr></thead><tbody><tr><td><code>type</code></td><td>Task type. Set the value to <code>segments</code>.</td><td>Yes</td></tr><tr><td><code>segments</code></td><td>A list of segment IDs.</td><td>Yes</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="compaction-dimensions-spec">Compaction dimensions spec<a href="#compaction-dimensions-spec" class="hash-link" aria-label="Direct link to Compaction dimensions spec" title="Direct link to Compaction dimensions spec"></a></h2><table><thead><tr><th>Field</th><th>Description</th><th>Required</th></tr></thead><tbody><tr><td><code>dimensions</code></td><td>A list of dimension names or objects. Cannot have the same column in both <code>dimensions</code> and <code>dimensionExclusions</code>. Defaults to <code>null</code>, which preserves the original dimensions.</td><td>No</td></tr><tr><td><code>dimensionExclusions</code></td><td>The names of dimensions to exclude from compaction. Only names are supported here, not objects. This list is only used if the dimensions list is null or empty; otherwise it is ignored. Defaults to <code>[]</code>.</td><td>No</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="compaction-transform-spec">Compaction transform spec<a href="#compaction-transform-spec" class="hash-link" aria-label="Direct link to Compaction transform spec" title="Direct link to Compaction transform spec"></a></h2><table><thead><tr><th>Field</th><th>Description</th><th>Required</th></tr></thead><tbody><tr><td><code>filter</code></td><td>The <code>filter</code> conditionally filters input rows during compaction. Only rows that pass the filter will be included in the compacted segments. Any of Druid's standard <a href="/docs/28.0.1/querying/filters">query filters</a> can be used. Defaults to 'null', which will not filter any row.</td><td>No</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="compaction-granularity-spec">Compaction granularity spec<a href="#compaction-granularity-spec" class="hash-link" aria-label="Direct link to Compaction granularity spec" title="Direct link to Compaction granularity spec"></a></h2><table><thead><tr><th>Field</th><th>Description</th><th>Required</th></tr></thead><tbody><tr><td><code>segmentGranularity</code></td><td>Time chunking period for the segment granularity. Defaults to 'null', which preserves the original segment granularity. Accepts all <a href="/docs/28.0.1/querying/granularities">Query granularity</a> values.</td><td>No</td></tr><tr><td><code>queryGranularity</code></td><td>The resolution of timestamp storage within each segment. Defaults to 'null', which preserves the original query granularity. Accepts all <a href="/docs/28.0.1/querying/granularities">Query granularity</a> values.</td><td>No</td></tr><tr><td><code>rollup</code></td><td>Enables compaction-time rollup. To preserve the original setting, keep the default value. To enable compaction-time rollup, set the value to <code>true</code>. Once the data is rolled up, you can no longer recover individual records.</td><td>No</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="learn-more">Learn more<a href="#learn-more" class="hash-link" aria-label="Direct link to Learn more" title="Direct link to Learn more"></a></h2><p>See the following topics for more information:</p><ul><li><a href="/docs/28.0.1/data-management/compaction">Compaction</a> for an overview of compaction and how to set up manual compaction in Druid.</li><li><a href="/docs/28.0.1/operations/segment-optimization">Segment optimization</a> for guidance on evaluating and optimizing Druid segment size.</li><li><a href="/docs/28.0.1/design/coordinator#automatic-compaction">Coordinator process</a> for details on how the Coordinator plans compaction tasks.</li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/docs/28.0.1/data-management/automatic-compaction"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Automatic compaction</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/docs/28.0.1/querying/sql"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Overview and syntax</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#setting-up-manual-compaction" class="table-of-contents__link toc-highlight">Setting up manual compaction</a></li><li><a href="#manual-compaction-task-example" class="table-of-contents__link toc-highlight">Manual compaction task example</a></li><li><a href="#compaction-io-configuration" class="table-of-contents__link toc-highlight">Compaction I/O configuration</a><ul><li><a href="#interval-inputspec" class="table-of-contents__link toc-highlight">Interval <code>inputSpec</code></a></li><li><a href="#segments-inputspec" class="table-of-contents__link toc-highlight">Segments <code>inputSpec</code></a></li></ul></li><li><a href="#compaction-dimensions-spec" class="table-of-contents__link toc-highlight">Compaction dimensions spec</a></li><li><a href="#compaction-transform-spec" class="table-of-contents__link toc-highlight">Compaction transform spec</a></li><li><a href="#compaction-granularity-spec" class="table-of-contents__link toc-highlight">Compaction granularity spec</a></li><li><a href="#learn-more" class="table-of-contents__link toc-highlight">Learn more</a></li></ul></div></div></div></div></main></div></div><footer class="footer"><div class="container container-fluid"><div class="footer__bottom text--center"><div class="margin-bottom--sm"><img src="/img/favicon.png" class="themedImage_ToTc themedImage--light_HNdA footer__logo"><img src="/img/favicon.png" class="themedImage_ToTc themedImage--dark_i4oU footer__logo"></div><div class="footer__copyright">Copyright © 2023 Apache Software Foundation. Except where otherwise noted, licensed under CC BY-SA 4.0. Apache Druid, Druid, and the Druid logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</div></div></div></footer></div> |
