| <!doctype html> |
| <html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-current plugin-docs plugin-id-default docs-doc-id-data-management/automatic-compaction"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="generator" content="Docusaurus v2.4.1"> |
| <title data-rh="true">Automatic compaction | Apache® Druid</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:image" content="https://druid.apache.org/img/druid_nav.png"><meta data-rh="true" name="twitter:image" content="https://druid.apache.org/img/druid_nav.png"><meta data-rh="true" property="og:url" content="https://druid.apache.org/docs/28.0.1/data-management/automatic-compaction"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="current"><meta data-rh="true" name="docusaurus_tag" content="docs-default-current"><meta data-rh="true" name="docsearch:version" content="current"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-current"><meta data-rh="true" property="og:title" content="Automatic compaction | Apache® Druid"><meta data-rh="true" name="description" content="<!--"><meta data-rh="true" property="og:description" content="<!--"><link data-rh="true" rel="icon" href="/img/favicon.png"><link data-rh="true" rel="canonical" href="https://druid.apache.org/docs/28.0.1/data-management/automatic-compaction"><link data-rh="true" rel="alternate" href="https://druid.apache.org/docs/28.0.1/data-management/automatic-compaction" hreflang="en"><link data-rh="true" rel="alternate" href="https://druid.apache.org/docs/28.0.1/data-management/automatic-compaction" hreflang="x-default"><link rel="preconnect" href="https://www.google-analytics.com"> |
| <link rel="preconnect" href="https://www.googletagmanager.com"> |
| <script async src="https://www.googletagmanager.com/gtag/js?id=UA-131010415-1"></script> |
| <script>function gtag(){dataLayer.push(arguments)}window.dataLayer=window.dataLayer||[],gtag("js",new Date),gtag("config","UA-131010415-1",{})</script> |
| |
| |
| |
| |
| <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"> |
| <script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><link rel="stylesheet" href="/assets/css/styles.546f39eb.css"> |
| <link rel="preload" href="/assets/js/runtime~main.a6b9689c.js" as="script"> |
| <link rel="preload" href="/assets/js/main.f7be26bf.js" as="script"> |
| </head> |
| <body class="navigation-with-keyboard"> |
| <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus"> |
| <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><nav aria-label="Main" class="navbar navbar--fixed-top navbar--dark"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/druid_nav.png" alt="Apache® Druid" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/druid_nav.png" alt="Apache® Druid" class="themedImage_ToTc themedImage--dark_i4oU"></div></a></div><div class="navbar__items navbar__items--right"><a class="navbar__item navbar__link" href="/technology">Technology</a><a class="navbar__item navbar__link" href="/use-cases">Use Cases</a><a class="navbar__item navbar__link" href="/druid-powered">Powered By</a><a class="navbar__item navbar__link" href="/docs/28.0.1/design/">Docs</a><a class="navbar__item navbar__link" href="/community/">Community</a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Apache®</a><ul class="dropdown__menu"><li><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Foundation<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://apachecon.com/?ref=druid.apache.org" target="_blank" rel="noopener noreferrer" class="dropdown__link">Events<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/licenses/" target="_blank" rel="noopener noreferrer" class="dropdown__link">License<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Thanks<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/security/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Security<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Sponsorship<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a class="navbar__item navbar__link" href="/downloads/">Download</a><div class="searchBox_ZlJk"><div class="navbar__search"><span aria-label="expand searchbar" role="button" class="search-icon" tabindex="0"></span><input type="search" id="search_input_react" placeholder="Loading..." aria-label="Search" class="navbar__search-input search-bar" disabled=""></div></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.1/design/">Getting started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.1/tutorials/tutorial-msq-extern">Tutorials</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.1/design/architecture">Design</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.1/ingestion/">Ingestion</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/docs/28.0.1/data-management/">Data management</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/28.0.1/data-management/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/28.0.1/data-management/update">Data updates</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/28.0.1/data-management/delete">Data deletion</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/28.0.1/data-management/schema-changes">Schema changes</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--active" aria-expanded="true" tabindex="0" href="/docs/28.0.1/data-management/compaction">Compaction</a><button aria-label="Toggle the collapsible sidebar category 'Compaction'" type="button" class="clean-btn menu__caret"></button></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/docs/28.0.1/data-management/automatic-compaction">Automatic compaction</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/28.0.1/data-management/manual-compaction">Manual compaction</a></li></ul></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.1/querying/sql">Querying</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.1/api-reference/">API reference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.1/configuration/">Configuration</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.1/operations/web-console">Operations</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.1/development/overview">Development</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.1/misc/papers-and-talks">Misc</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.1/release-info/release-notes">Release info</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Data management</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item"><a class="breadcrumbs__link" itemprop="item" href="/docs/28.0.1/data-management/compaction"><span itemprop="name">Compaction</span></a><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Automatic compaction</span><meta itemprop="position" content="3"></li></ul></nav><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Automatic compaction</h1></header><p>In Apache Druid, compaction is a special type of ingestion task that reads data from a Druid datasource and writes it back into the same datasource. A common use case for this is to <a href="/docs/28.0.1/operations/segment-optimization">optimally size segments</a> after ingestion to improve query performance. Automatic compaction, or auto-compaction, refers to the system for automatic execution of compaction tasks managed by the <a href="/docs/28.0.1/design/coordinator">Druid Coordinator</a>. |
| This topic guides you through setting up automatic compaction for your Druid cluster. See the <a href="#examples">examples</a> for common use cases for automatic compaction.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="how-druid-manages-automatic-compaction">How Druid manages automatic compaction<a href="#how-druid-manages-automatic-compaction" class="hash-link" aria-label="Direct link to How Druid manages automatic compaction" title="Direct link to How Druid manages automatic compaction"></a></h2><p>The Coordinator <a href="/docs/28.0.1/configuration/#coordinator-operation">indexing period</a>, <code>druid.coordinator.period.indexingPeriod</code>, controls the frequency of compaction tasks. |
| The default indexing period is 30 minutes, meaning that the Coordinator first checks for segments to compact at most 30 minutes from when auto-compaction is enabled. |
| This time period affects other Coordinator duties including merge and conversion tasks. |
| To configure the auto-compaction time period without interfering with <code>indexingPeriod</code>, see <a href="#set-frequency-of-compaction-runs">Set frequency of compaction runs</a>.</p><p>At every invocation of auto-compaction, the Coordinator initiates a <a href="/docs/28.0.1/design/coordinator#segment-search-policy-in-automatic-compaction">segment search</a> to determine eligible segments to compact. |
| When there are eligible segments to compact, the Coordinator issues compaction tasks based on available worker capacity. |
| If a compaction task takes longer than the indexing period, the Coordinator waits for it to finish before resuming the period for segment search.</p><div class="theme-admonition theme-admonition-info alert alert--info admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg></span>info</div><div class="admonitionContent_S0QG"><p> Auto-compaction skips datasources that have a segment granularity of <code>ALL</code>.</p></div></div><p>As a best practice, you should set up auto-compaction for all Druid datasources. You can run compaction tasks manually for cases where you want to allocate more system resources. For example, you may choose to run multiple compaction tasks in parallel to compact an existing datasource for the first time. See <a href="/docs/28.0.1/data-management/compaction">Compaction</a> for additional details and use cases.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="enable-automatic-compaction">Enable automatic compaction<a href="#enable-automatic-compaction" class="hash-link" aria-label="Direct link to Enable automatic compaction" title="Direct link to Enable automatic compaction"></a></h2><p>You can enable automatic compaction for a datasource using the web console or programmatically via an API. |
| This process differs for manual compaction tasks, which can be submitted from the <a href="/docs/28.0.1/operations/web-console">Tasks view of the web console</a> or the <a href="/docs/28.0.1/api-reference/tasks-api">Tasks API</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="web-console">Web console<a href="#web-console" class="hash-link" aria-label="Direct link to Web console" title="Direct link to Web console"></a></h3><p>Use the web console to enable automatic compaction for a datasource as follows.</p><ol><li>Click <strong>Datasources</strong> in the top-level navigation.</li><li>In the <strong>Compaction</strong> column, click the edit icon for the datasource to compact.</li><li>In the <strong>Compaction config</strong> dialog, configure the auto-compaction settings. The dialog offers a form view as well as a JSON view. Editing the form updates the JSON specification, and editing the JSON updates the form field, if present. Form fields not present in the JSON indicate default values. You may add additional properties to the JSON for auto-compaction settings not displayed in the form. See <a href="#configure-automatic-compaction">Configure automatic compaction</a> for supported settings for auto-compaction.</li><li>Click <strong>Submit</strong>.</li><li>Refresh the <strong>Datasources</strong> view. The <strong>Compaction</strong> column for the datasource changes from “Not enabled” to “Awaiting first run.”</li></ol><p>The following screenshot shows the compaction config dialog for a datasource with auto-compaction enabled. |
| <img loading="lazy" alt="Compaction config in web console" src="/assets/images/compaction-config-2d5e391fe681a9c978060c383357b531.png" width="1250" height="640" class="img_ev3q"></p><p>To disable auto-compaction for a datasource, click <strong>Delete</strong> from the <strong>Compaction config</strong> dialog. Druid does not retain your auto-compaction configuration.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="compaction-configuration-api">Compaction configuration API<a href="#compaction-configuration-api" class="hash-link" aria-label="Direct link to Compaction configuration API" title="Direct link to Compaction configuration API"></a></h3><p>Use the <a href="/docs/28.0.1/api-reference/automatic-compaction-api#manage-automatic-compaction">Automatic compaction API</a> to configure automatic compaction. |
| To enable auto-compaction for a datasource, create a JSON object with the desired auto-compaction settings. |
| See <a href="#configure-automatic-compaction">Configure automatic compaction</a> for the syntax of an auto-compaction spec. |
| Send the JSON object as a payload in a <a href="/docs/28.0.1/api-reference/automatic-compaction-api#create-or-update-automatic-compaction-configuration"><code>POST</code> request</a> to <code>/druid/coordinator/v1/config/compaction</code>. |
| The following example configures auto-compaction for the <code>wikipedia</code> datasource:</p><div class="language-sh codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sh codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">curl --location --request POST 'http://localhost:8081/druid/coordinator/v1/config/compaction' \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--header 'Content-Type: application/json' \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--data-raw '{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "dataSource": "wikipedia",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "granularitySpec": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "segmentGranularity": "DAY"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">}'</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>To disable auto-compaction for a datasource, send a <a href="/docs/28.0.1/api-reference/automatic-compaction-api#remove-automatic-compaction-configuration"><code>DELETE</code> request</a> to <code>/druid/coordinator/v1/config/compaction/{dataSource}</code>. Replace <code>{dataSource}</code> with the name of the datasource for which to disable auto-compaction. For example:</p><div class="language-sh codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sh codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">curl --location --request DELETE 'http://localhost:8081/druid/coordinator/v1/config/compaction/wikipedia'</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="configure-automatic-compaction">Configure automatic compaction<a href="#configure-automatic-compaction" class="hash-link" aria-label="Direct link to Configure automatic compaction" title="Direct link to Configure automatic compaction"></a></h2><p>You can configure automatic compaction dynamically without restarting Druid. |
| The automatic compaction system uses the following syntax:</p><div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"dataSource"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> <task_datasource></span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"ioConfig"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> <IO config></span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"dimensionsSpec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> <custom dimensionsSpec></span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"transformSpec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> <custom transformSpec></span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"metricsSpec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> <custom metricsSpec></span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"tuningConfig"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> <parallel indexing task tuningConfig></span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"granularitySpec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> <compaction task granularitySpec></span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"skipOffsetFromLatest"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> <time period to avoid compaction></span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"taskPriority"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> <compaction task priority></span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"taskContext"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> <task context></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Most fields in the auto-compaction configuration correlate to a typical <a href="/docs/28.0.1/ingestion/ingestion-spec">Druid ingestion spec</a>. |
| The following properties only apply to auto-compaction:</p><ul><li><code>skipOffsetFromLatest</code></li><li><code>taskPriority</code></li><li><code>taskContext</code></li></ul><p>Since the automatic compaction system provides a management layer on top of manual compaction tasks, |
| the auto-compaction configuration does not include task-specific properties found in a typical Druid ingestion spec. |
| The following properties are automatically set by the Coordinator:</p><ul><li><code>type</code>: Set to <code>compact</code>.</li><li><code>id</code>: Generated using the task type, datasource name, interval, and timestamp. The task ID is prefixed with <code>coordinator-issued</code>.</li><li><code>context</code>: Set according to the user-provided <code>taskContext</code>.</li></ul><p>Compaction tasks typically fetch all <a href="/docs/28.0.1/data-management/compaction#compaction-io-configuration">relevant segments</a> prior to launching any subtasks, |
| <em>unless</em> the following properties are all set to non-null values. It is strongly recommended to set them to non-null values to |
| maximize performance and minimize disk usage of the <code>compact</code> tasks launched by auto-compaction:</p><ul><li><a href="/docs/28.0.1/data-management/compaction#compaction-granularity-spec"><code>granularitySpec</code></a>, with non-null values for each of <code>segmentGranularity</code>, <code>queryGranularity</code>, and <code>rollup</code></li><li><a href="/docs/28.0.1/data-management/compaction#compaction-dimensions-spec"><code>dimensionsSpec</code></a></li><li><code>metricsSpec</code></li></ul><p>For more details on each of the specs in an auto-compaction configuration, see <a href="/docs/28.0.1/configuration/#automatic-compaction-dynamic-configuration">Automatic compaction dynamic configuration</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="avoid-conflicts-with-ingestion">Avoid conflicts with ingestion<a href="#avoid-conflicts-with-ingestion" class="hash-link" aria-label="Direct link to Avoid conflicts with ingestion" title="Direct link to Avoid conflicts with ingestion"></a></h3><p>Compaction tasks may be interrupted when they interfere with ingestion. For example, this occurs when an ingestion task needs to write data to a segment for a time interval locked for compaction. If there are continuous failures that prevent compaction from making progress, consider one of the following strategies:</p><ul><li>Set <code>skipOffsetFromLatest</code> to reduce the chance of conflicts between ingestion and compaction. See more details in this section below.</li><li>Increase the priority value of compaction tasks relative to ingestion tasks. Only recommended for advanced users. This approach can cause ingestion jobs to fail or lag. To change the priority of compaction tasks, set <code>taskPriority</code> to the desired priority value in the auto-compaction configuration. For details on the priority values of different task types, see <a href="/docs/28.0.1/ingestion/tasks#lock-priority">Lock priority</a>.</li></ul><p>The Coordinator compacts segments from newest to oldest. In the auto-compaction configuration, you can set a time period, relative to the end time of the most recent segment, for segments that should not be compacted. Assign this value to <code>skipOffsetFromLatest</code>. Note that this offset is not relative to the current time but to the latest segment time. For example, if you want to skip over segments from five days prior to the end time of the most recent segment, assign <code>"skipOffsetFromLatest": "P5D"</code>.</p><p>To set <code>skipOffsetFromLatest</code>, consider how frequently you expect the stream to receive late arriving data. If your stream only occasionally receives late arriving data, the auto-compaction system robustly compacts your data even though data is ingested outside the <code>skipOffsetFromLatest</code> window. For most realtime streaming ingestion use cases, it is reasonable to set <code>skipOffsetFromLatest</code> to a few hours or a day.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="set-frequency-of-compaction-runs">Set frequency of compaction runs<a href="#set-frequency-of-compaction-runs" class="hash-link" aria-label="Direct link to Set frequency of compaction runs" title="Direct link to Set frequency of compaction runs"></a></h3><p>If you want the Coordinator to check for compaction more frequently than its indexing period, create a separate group to handle compaction duties. |
| Set the time period of the duty group in the <code>coordinator/runtime.properties</code> file. |
| The following example shows how to create a duty group named <code>compaction</code> and set the auto-compaction period to 1 minute:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.coordinator.dutyGroups=["compaction"]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.coordinator.compaction.duties=["compactSegments"]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.coordinator.compaction.period=PT60S</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="view-automatic-compaction-statistics">View automatic compaction statistics<a href="#view-automatic-compaction-statistics" class="hash-link" aria-label="Direct link to View automatic compaction statistics" title="Direct link to View automatic compaction statistics"></a></h2><p>After the Coordinator has initiated auto-compaction, you can view compaction statistics for the datasource, including the number of bytes, segments, and intervals already compacted and those awaiting compaction. The Coordinator also reports the total bytes, segments, and intervals not eligible for compaction in accordance with its <a href="/docs/28.0.1/design/coordinator#segment-search-policy-in-automatic-compaction">segment search policy</a>.</p><p>In the web console, the Datasources view displays auto-compaction statistics. The Tasks view shows the task information for compaction tasks that were triggered by the automatic compaction system.</p><p>To get statistics by API, send a <a href="/docs/28.0.1/api-reference/automatic-compaction-api#view-automatic-compaction-status"><code>GET</code> request</a> to <code>/druid/coordinator/v1/compaction/status</code>. To filter the results to a particular datasource, pass the datasource name as a query parameter to the request—for example, <code>/druid/coordinator/v1/compaction/status?dataSource=wikipedia</code>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="examples">Examples<a href="#examples" class="hash-link" aria-label="Direct link to Examples" title="Direct link to Examples"></a></h2><p>The following examples demonstrate potential use cases in which auto-compaction may improve your Druid performance. See more details in <a href="/docs/28.0.1/data-management/compaction#compaction-guidelines">Compaction strategies</a>. The examples in this section do not change the underlying data.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="change-segment-granularity">Change segment granularity<a href="#change-segment-granularity" class="hash-link" aria-label="Direct link to Change segment granularity" title="Direct link to Change segment granularity"></a></h3><p>You have a stream set up to ingest data with <code>HOUR</code> segment granularity into the <code>wikistream</code> datasource. You notice that your Druid segments are smaller than the <a href="/docs/28.0.1/operations/segment-optimization">recommended segment size</a> of 5 million rows per segment. You wish to automatically compact segments to <code>DAY</code> granularity while leaving the latest week of data <em>not</em> compacted because your stream consistently receives data within that time period.</p><p>The following auto-compaction configuration compacts existing <code>HOUR</code> segments into <code>DAY</code> segments while leaving the latest week of data not compacted:</p><div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"dataSource"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"wikistream"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"granularitySpec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"segmentGranularity"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"DAY"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"skipOffsetFromLatest"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"P1W"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="update-partitioning-scheme">Update partitioning scheme<a href="#update-partitioning-scheme" class="hash-link" aria-label="Direct link to Update partitioning scheme" title="Direct link to Update partitioning scheme"></a></h3><p>For your <code>wikipedia</code> datasource, you want to optimize segment access when regularly ingesting data without compromising compute time when querying the data. Your ingestion spec for batch append uses <a href="/docs/28.0.1/ingestion/native-batch#dynamic-partitioning">dynamic partitioning</a> to optimize for write-time operations, while your stream ingestion partitioning is configured by the stream service. You want to implement auto-compaction to reorganize the data with a suitable read-time partitioning using <a href="/docs/28.0.1/ingestion/native-batch#multi-dimension-range-partitioning">multi-dimension range partitioning</a>. Based on the dimensions frequently accessed in queries, you wish to partition on the following dimensions: <code>channel</code>, <code>countryName</code>, <code>namespace</code>.</p><p>The following auto-compaction configuration compacts updates the <code>wikipedia</code> segments to use multi-dimension range partitioning:</p><div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"dataSource"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"wikipedia"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"tuningConfig"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"partitionsSpec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"range"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"partitionDimensions"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"channel"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"countryName"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"namespace"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"targetRowsPerSegment"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">5000000</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="concurrent-append-and-replace">Concurrent append and replace<a href="#concurrent-append-and-replace" class="hash-link" aria-label="Direct link to Concurrent append and replace" title="Direct link to Concurrent append and replace"></a></h2><div class="theme-admonition theme-admonition-info alert alert--info admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg></span>info</div><div class="admonitionContent_S0QG"><p>Concurrent append and replace is an <a href="/docs/28.0.1/development/experimental">experimental feature</a> and is not currently available for SQL-based ingestion.</p></div></div><p>This feature allows you to safely replace the existing data in an interval of a datasource while new data is being appended to that interval. One of the most common applications of this is appending new data (using say streaming ingestion) to an interval while compaction of that interval is already in progress.</p><p>To set up concurrent append and replace, you need to ensure that your ingestion jobs have the appropriate lock types:</p><p>You can enable concurrent append and replace by ensuring the following:</p><ul><li>The append task (with <code>appendToExisting</code> set to <code>true</code>) has <code>taskLockType</code> set to <code>APPEND</code> in the task context.</li><li>The replace task (with <code>appendToExisting</code> set to <code>false</code>) has <code>taskLockType</code> set to <code>REPLACE</code> in the task context.</li><li>The segment granularity of the append task is equal to or finer than the segment granularity of the replace task.</li></ul><div class="theme-admonition theme-admonition-info alert alert--info admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg></span>info</div><div class="admonitionContent_S0QG"><p>When using concurrent append and replace, keep the following in mind:</p><ul><li><p>Concurrent append and replace fails if the task with <code>APPEND</code> lock uses a coarser segment granularity than the task with the <code>REPLACE</code> lock. For example, if the <code>APPEND</code> task uses a segment granularity of YEAR and the <code>REPLACE</code> task uses a segment granularity of MONTH, you should not use concurrent append and replace.</p></li><li><p>Only a single task can hold a <code>REPLACE</code> lock on a given interval of a datasource.</p></li><li><p>Multiple tasks can hold <code>APPEND</code> locks on a given interval of a datasource and append data to that interval simultaneously.</p></li></ul></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="configure-concurrent-append-and-replace">Configure concurrent append and replace<a href="#configure-concurrent-append-and-replace" class="hash-link" aria-label="Direct link to Configure concurrent append and replace" title="Direct link to Configure concurrent append and replace"></a></h3><h5 class="anchor anchorWithStickyNavbar_LWe7" id="update-the-compaction-settings-with-the-api">Update the compaction settings with the API<a href="#update-the-compaction-settings-with-the-api" class="hash-link" aria-label="Direct link to Update the compaction settings with the API" title="Direct link to Update the compaction settings with the API"></a></h5><p> Prepare your datasource for concurrent append and replace by setting its task lock type to <code>REPLACE</code>. |
| Add the <code>taskContext</code> like you would any other automatic compaction setting through the API:</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token function" style="color:rgb(130, 170, 255)">curl</span><span class="token plain"> --location --request POST </span><span class="token string" style="color:rgb(195, 232, 141)">'http://localhost:8081/druid/coordinator/v1/config/compaction'</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">\</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--header </span><span class="token string" style="color:rgb(195, 232, 141)">'Content-Type: application/json'</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">\</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--data-raw </span><span class="token string" style="color:rgb(195, 232, 141)">'{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> "dataSource": "YOUR_DATASOURCE",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> "taskContext": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> "taskLockType": "REPLACE"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token string" style="color:rgb(195, 232, 141)">}'</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h5 class="anchor anchorWithStickyNavbar_LWe7" id="update-the-compaction-settings-with-the-ui">Update the compaction settings with the UI<a href="#update-the-compaction-settings-with-the-ui" class="hash-link" aria-label="Direct link to Update the compaction settings with the UI" title="Direct link to Update the compaction settings with the UI"></a></h5><p>In the <strong>Compaction config</strong> for a datasource, set <strong>Allow concurrent compactions (experimental)</strong> to <strong>True</strong>.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="add-a-task-lock-type-to-your-ingestion-job">Add a task lock type to your ingestion job<a href="#add-a-task-lock-type-to-your-ingestion-job" class="hash-link" aria-label="Direct link to Add a task lock type to your ingestion job" title="Direct link to Add a task lock type to your ingestion job"></a></h4><p>Next, you need to configure the task lock type for your ingestion job: </p><ul><li>For streaming jobs, the context parameter goes in your supervisor spec, and the lock type is always <code>APPEND</code></li><li>For legacy JSON-based batch ingestion, the context parameter goes in your ingestion spec, and the lock type can be either <code>APPEND</code> or <code>REPLACE</code>. </li></ul><p>You can provide the context parameter through the API like any other parameter for ingestion job or through the UI.</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="add-the-task-lock-type-through-the-api">Add the task lock type through the API<a href="#add-the-task-lock-type-through-the-api" class="hash-link" aria-label="Direct link to Add the task lock type through the API" title="Direct link to Add the task lock type through the API"></a></h5><p>Add the following JSON snippet to your supervisor or ingestion spec if you're using the API:</p><div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token property">"context"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"taskLockType"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> LOCK_TYPE</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"> </span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The <code>LOCK_TYPE</code> depends on what you're trying to accomplish.</p><p>Set <code>taskLockType</code> to <code>APPEND</code> if either of the following are true:</p><ul><li>Dynamic partitioning with append to existing is set to <code>true</code></li><li>The ingestion job is a streaming ingestion job</li></ul><p>If you have multiple ingestion jobs that append all targeting the same datasource and want them to run simultaneously, you need to also include the following context parameter:</p><div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token property">"useSharedLock"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"true"</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Keep in mind that <code>taskLockType</code> takes precedence over <code>useSharedLock</code>. Do not use it with <code>REPLACE</code> task locks.</p><p>Set <code>taskLockType</code> to <code>REPLACE</code> if you're replacing data. For example, if you use any of the following partitioning types, use <code>REPLACE</code>:</p><ul><li>hash partitioning </li><li>range partitioning</li><li>dynamic partitioning with append to existing set to <code>false</code></li></ul><h5 class="anchor anchorWithStickyNavbar_LWe7" id="add-a-task-lock-using-the-druid-console">Add a task lock using the Druid console<a href="#add-a-task-lock-using-the-druid-console" class="hash-link" aria-label="Direct link to Add a task lock using the Druid console" title="Direct link to Add a task lock using the Druid console"></a></h5><p>As part of the <strong>Load data</strong> wizard for classic batch (JSON-based ingestion) and streaming ingestion, you can configure the task lock type for the ingestion during the <strong>Publish</strong> step:</p><ul><li>If you set <strong>Append to existing</strong> to <strong>True</strong>, you can then set <strong>Allow concurrent append tasks (experimental)</strong> to <strong>True</strong>.</li><li>If you set <strong>Append to existing</strong> to <strong>False</strong>, you can then set <strong>Allow concurrent replace tasks (experimental)</strong> to <strong>True</strong>.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="learn-more">Learn more<a href="#learn-more" class="hash-link" aria-label="Direct link to Learn more" title="Direct link to Learn more"></a></h2><p>See the following topics for more information:</p><ul><li><a href="/docs/28.0.1/data-management/compaction">Compaction</a> for an overview of compaction and how to set up manual compaction in Druid.</li><li><a href="/docs/28.0.1/operations/segment-optimization">Segment optimization</a> for guidance on evaluating and optimizing Druid segment size.</li><li><a href="/docs/28.0.1/design/coordinator#automatic-compaction">Coordinator process</a> for details on how the Coordinator plans compaction tasks.</li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/docs/28.0.1/data-management/compaction"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Compaction</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/docs/28.0.1/data-management/manual-compaction"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Manual compaction</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#how-druid-manages-automatic-compaction" class="table-of-contents__link toc-highlight">How Druid manages automatic compaction</a></li><li><a href="#enable-automatic-compaction" class="table-of-contents__link toc-highlight">Enable automatic compaction</a><ul><li><a href="#web-console" class="table-of-contents__link toc-highlight">Web console</a></li><li><a href="#compaction-configuration-api" class="table-of-contents__link toc-highlight">Compaction configuration API</a></li></ul></li><li><a href="#configure-automatic-compaction" class="table-of-contents__link toc-highlight">Configure automatic compaction</a><ul><li><a href="#avoid-conflicts-with-ingestion" class="table-of-contents__link toc-highlight">Avoid conflicts with ingestion</a></li><li><a href="#set-frequency-of-compaction-runs" class="table-of-contents__link toc-highlight">Set frequency of compaction runs</a></li></ul></li><li><a href="#view-automatic-compaction-statistics" class="table-of-contents__link toc-highlight">View automatic compaction statistics</a></li><li><a href="#examples" class="table-of-contents__link toc-highlight">Examples</a><ul><li><a href="#change-segment-granularity" class="table-of-contents__link toc-highlight">Change segment granularity</a></li><li><a href="#update-partitioning-scheme" class="table-of-contents__link toc-highlight">Update partitioning scheme</a></li></ul></li><li><a href="#concurrent-append-and-replace" class="table-of-contents__link toc-highlight">Concurrent append and replace</a><ul><li><a href="#configure-concurrent-append-and-replace" class="table-of-contents__link toc-highlight">Configure concurrent append and replace</a></li></ul></li><li><a href="#learn-more" class="table-of-contents__link toc-highlight">Learn more</a></li></ul></div></div></div></div></main></div></div><footer class="footer"><div class="container container-fluid"><div class="footer__bottom text--center"><div class="margin-bottom--sm"><img src="/img/favicon.png" class="themedImage_ToTc themedImage--light_HNdA footer__logo"><img src="/img/favicon.png" class="themedImage_ToTc themedImage--dark_i4oU footer__logo"></div><div class="footer__copyright">Copyright © 2023 Apache Software Foundation. Except where otherwise noted, licensed under CC BY-SA 4.0. Apache Druid, Druid, and the Druid logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</div></div></div></footer></div> |
| <script src="/assets/js/runtime~main.a6b9689c.js"></script> |
| <script src="/assets/js/main.f7be26bf.js"></script> |
| </body> |
| </html> |
