| <!doctype html> |
| <html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-current plugin-docs plugin-id-default docs-doc-id-operations/clean-metadata-store"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="generator" content="Docusaurus v2.4.1"> |
| <title data-rh="true">Automated cleanup for metadata records | Apache® Druid</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:image" content="https://druid.apache.org/img/druid_nav.png"><meta data-rh="true" name="twitter:image" content="https://druid.apache.org/img/druid_nav.png"><meta data-rh="true" property="og:url" content="https://druid.apache.org/docs/latest/operations/clean-metadata-store"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="current"><meta data-rh="true" name="docusaurus_tag" content="docs-default-current"><meta data-rh="true" name="docsearch:version" content="current"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-current"><meta data-rh="true" property="og:title" content="Automated cleanup for metadata records | Apache® Druid"><meta data-rh="true" name="description" content="Defines a strategy to maintain Druid metadata store performance by automatically removing leftover records for deleted entities: datasources, supervisors, rules, compaction configuration, audit records, etc. Most applicable to databases with 'high-churn' datasources."><meta data-rh="true" property="og:description" content="Defines a strategy to maintain Druid metadata store performance by automatically removing leftover records for deleted entities: datasources, supervisors, rules, compaction configuration, audit records, etc. Most applicable to databases with 'high-churn' datasources."><link data-rh="true" rel="icon" href="/img/favicon.png"><link data-rh="true" rel="canonical" href="https://druid.apache.org/docs/latest/operations/clean-metadata-store"><link data-rh="true" rel="alternate" href="https://druid.apache.org/docs/latest/operations/clean-metadata-store" hreflang="en"><link data-rh="true" rel="alternate" href="https://druid.apache.org/docs/latest/operations/clean-metadata-store" hreflang="x-default"><link rel="preconnect" href="https://www.google-analytics.com"> |
| <link rel="preconnect" href="https://www.googletagmanager.com"> |
| <script async src="https://www.googletagmanager.com/gtag/js?id=UA-131010415-1"></script> |
| <script>function gtag(){dataLayer.push(arguments)}window.dataLayer=window.dataLayer||[],gtag("js",new Date),gtag("config","UA-131010415-1",{})</script> |
| |
| |
| |
| |
| <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"> |
| <script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><link rel="stylesheet" href="/assets/css/styles.546f39eb.css"> |
| <link rel="preload" href="/assets/js/runtime~main.26d714fb.js" as="script"> |
| <link rel="preload" href="/assets/js/main.bd54ee66.js" as="script"> |
| </head> |
| <body class="navigation-with-keyboard"> |
| <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus"> |
| <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><nav aria-label="Main" class="navbar navbar--fixed-top navbar--dark"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/druid_nav.png" alt="Apache® Druid" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/druid_nav.png" alt="Apache® Druid" class="themedImage_ToTc themedImage--dark_i4oU"></div></a></div><div class="navbar__items navbar__items--right"><a class="navbar__item navbar__link" href="/technology">Technology</a><a class="navbar__item navbar__link" href="/use-cases">Use Cases</a><a class="navbar__item navbar__link" href="/druid-powered">Powered By</a><a class="navbar__item navbar__link" href="/docs/latest/design/">Docs</a><a class="navbar__item navbar__link" href="/community/">Community</a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Apache®</a><ul class="dropdown__menu"><li><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Foundation<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://apachecon.com/?ref=druid.apache.org" target="_blank" rel="noopener noreferrer" class="dropdown__link">Events<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/licenses/" target="_blank" rel="noopener noreferrer" class="dropdown__link">License<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Thanks<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/security/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Security<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Sponsorship<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a class="navbar__item navbar__link" href="/downloads/">Download</a><div class="searchBox_ZlJk"><div class="navbar__search"><span aria-label="expand searchbar" role="button" class="search-icon" tabindex="0"></span><input type="search" id="search_input_react" placeholder="Loading..." aria-label="Search" class="navbar__search-input search-bar" disabled=""></div></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/latest/design/">Getting started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/latest/tutorials/tutorial-msq-extern">Tutorials</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/latest/design/architecture">Design</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/latest/ingestion/">Ingestion</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/latest/data-management/">Data management</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/latest/querying/sql">Querying</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/latest/api-reference/">API reference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/latest/configuration/">Configuration</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/docs/latest/operations/web-console">Operations</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/latest/operations/web-console">Web console</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/latest/operations/java">Java runtime</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/latest/operations/durable-storage">Durable storage</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/latest/operations/security-overview">Security</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/docs/latest/operations/basic-cluster-tuning">Performance tuning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/latest/operations/basic-cluster-tuning">Basic cluster tuning</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/latest/operations/segment-optimization">Segment size optimization</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/latest/operations/mixed-workloads">Mixed workloads</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/latest/operations/http-compression">HTTP compression</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/docs/latest/operations/clean-metadata-store">Automated metadata cleanup</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/latest/operations/request-logging">Monitoring</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/latest/operations/high-availability">High availability</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/latest/operations/rolling-updates">Rolling updates</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/latest/operations/rule-configuration">Using rules to drop and retain data</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/latest/operations/migrate-from-firehose">Migrate from firehose</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/latest/operations/other-hadoop">Working with different versions of Apache Hadoop</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/latest/operations/dump-segment">Misc</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/latest/development/overview">Development</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/latest/misc/papers-and-talks">Misc</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Operations</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Performance tuning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Automated metadata cleanup</span><meta itemprop="position" content="3"></li></ul></nav><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Automated cleanup for metadata records</h1></header><p>Apache Druid relies on <a href="/docs/latest/design/metadata-storage">metadata storage</a> to track information on data storage, operations, and system configuration. |
| The metadata store includes the following:</p><ul><li>Segment records</li><li>Audit records</li><li>Supervisor records</li><li>Rule records</li><li>Compaction configuration records</li><li>Datasource records created by supervisors</li><li>Indexer task logs</li></ul><p>When you delete some entities from Apache Druid, records related to the entity may remain in the metadata store. |
| If you have a high datasource churn rate, meaning you frequently create and delete many short-lived datasources or other related entities like compaction configuration or rules, the leftover records can fill your metadata store and cause performance issues. |
| To maintain metadata store performance, you can configure Apache Druid to automatically remove records associated with deleted entities from the metadata store.</p><p>By default, Druid automatically cleans up metadata older than 90 days. |
| This applies to all metadata entities in this topic except compaction configuration records and indexer task logs, for which cleanup is disabled by default. |
| You can configure the retention period for each metadata type, when available, through the record's <code>durationToRetain</code> property. |
| Certain records may require additional conditions be satisfied before clean up occurs.</p><p>See the <a href="#example">example</a> for how you can customize the automated metadata cleanup for a specific use case.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="automated-cleanup-strategies">Automated cleanup strategies<a href="#automated-cleanup-strategies" class="hash-link" aria-label="Direct link to Automated cleanup strategies" title="Direct link to Automated cleanup strategies"></a></h2><p>There are several cases when you should consider automated cleanup of the metadata related to deleted datasources:</p><ul><li>If you know you have many high-churn datasources, for example, you have scripts that create and delete supervisors regularly.</li><li>If you have issues with the hard disk for your metadata database filling up.</li><li>If you run into performance issues with the metadata database. For example, API calls are very slow or fail to execute.</li></ul><p>If you have compliance requirements to keep audit records and you enable automated cleanup for audit records, use alternative methods to preserve audit metadata, for example, by periodically exporting audit metadata records to external storage.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="configure-automated-metadata-cleanup">Configure automated metadata cleanup<a href="#configure-automated-metadata-cleanup" class="hash-link" aria-label="Direct link to Configure automated metadata cleanup" title="Direct link to Configure automated metadata cleanup"></a></h2><p>You can configure cleanup for each entity separately, as described in this section. |
| Define the properties in the <code>coordinator/runtime.properties</code> file.</p><p>The cleanup of one entity may depend on the cleanup of another entity as follows:</p><ul><li>You have to configure a <a href="#kill-task">kill task for segment records</a> before you can configure automated cleanup for <a href="#rules-records">rules</a> or <a href="#compaction-configuration-records">compaction configuration</a>.</li><li>You have to schedule the metadata management tasks to run at the same or higher frequency as your most frequent cleanup job. For example, if your most frequent cleanup job is every hour, set the metadata store management period to one hour or less: <code>druid.coordinator.period.metadataStoreManagementPeriod=P1H</code>.</li></ul><p>For details on configuration properties, see <a href="/docs/latest/configuration/#metadata-management">Metadata management</a>. |
| If you want to skip the details, check out the <a href="#example">example</a> for configuring automated metadata cleanup.</p><a name="kill-task"></a><h3 class="anchor anchorWithStickyNavbar_LWe7" id="segment-records-and-segments-in-deep-storage-kill-task">Segment records and segments in deep storage (kill task)<a href="#segment-records-and-segments-in-deep-storage-kill-task" class="hash-link" aria-label="Direct link to Segment records and segments in deep storage (kill task)" title="Direct link to Segment records and segments in deep storage (kill task)"></a></h3><div class="theme-admonition theme-admonition-info alert alert--info admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg></span>info</div><div class="admonitionContent_S0QG"><p> The kill task is the only configuration in this topic that affects actual data in deep storage and not simply metadata or logs.</p></div></div><p>Segment records and segments in deep storage become eligible for deletion when both of the following conditions hold:</p><ul><li>When they meet the eligibility requirement of kill task datasource configuration according to <code>killDataSourceWhitelist</code> set in the Coordinator dynamic configuration. See <a href="/docs/latest/configuration/#dynamic-configuration">Dynamic configuration</a>.</li><li>When the <code>durationToRetain</code> time has passed since their creation.</li></ul><p>Kill tasks use the following configuration:</p><ul><li><code>druid.coordinator.kill.on</code>: When <code>true</code>, enables the Coordinator to submit a kill task for unused segments, which deletes them completely from metadata store and from deep storage. |
| Only applies to the specified datasources in the dynamic configuration parameter <code>killDataSourceWhitelist</code>. |
| If <code>killDataSourceWhitelist</code> is not set or empty, then kill tasks can be submitted for all datasources.</li><li><code>druid.coordinator.kill.period</code>: Defines the frequency in <a href="https://en.wikipedia.org/wiki/ISO_8601#Durations" target="_blank" rel="noopener noreferrer">ISO 8601 format</a> for the cleanup job to check for and delete eligible segments. Defaults to <code>P1D</code>. Must be greater than <code>druid.coordinator.period.indexingPeriod</code>. </li><li><code>druid.coordinator.kill.durationToRetain</code>: Defines the retention period in <a href="https://en.wikipedia.org/wiki/ISO_8601#Durations" target="_blank" rel="noopener noreferrer">ISO 8601 format</a> after creation that segments become eligible for deletion.</li><li><code>druid.coordinator.kill.maxSegments</code>: Defines the maximum number of segments to delete per kill task.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="audit-records">Audit records<a href="#audit-records" class="hash-link" aria-label="Direct link to Audit records" title="Direct link to Audit records"></a></h3><p>All audit records become eligible for deletion when the <code>durationToRetain</code> time has passed since their creation.</p><p>Audit cleanup uses the following configuration:</p><ul><li><code>druid.coordinator.kill.audit.on</code>: When <code>true</code>, enables cleanup for audit records.</li><li><code>druid.coordinator.kill.audit.period</code>: Defines the frequency in <a href="https://en.wikipedia.org/wiki/ISO_8601#Durations" target="_blank" rel="noopener noreferrer">ISO 8601 format</a> for the cleanup job to check for and delete eligible audit records. Defaults to <code>P1D</code>.</li><li><code>druid.coordinator.kill.audit.durationToRetain</code>: Defines the retention period in <a href="https://en.wikipedia.org/wiki/ISO_8601#Durations" target="_blank" rel="noopener noreferrer">ISO 8601 format</a> after creation that audit records become eligible for deletion.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="supervisor-records">Supervisor records<a href="#supervisor-records" class="hash-link" aria-label="Direct link to Supervisor records" title="Direct link to Supervisor records"></a></h3><p>Supervisor records become eligible for deletion when the supervisor is terminated and the <code>durationToRetain</code> time has passed since their creation.</p><p>Supervisor cleanup uses the following configuration:</p><ul><li><code>druid.coordinator.kill.supervisor.on</code>: When <code>true</code>, enables cleanup for supervisor records.</li><li><code>druid.coordinator.kill.supervisor.period</code>: Defines the frequency in <a href="https://en.wikipedia.org/wiki/ISO_8601#Durations" target="_blank" rel="noopener noreferrer">ISO 8601 format</a> for the cleanup job to check for and delete eligible supervisor records. Defaults to <code>P1D</code>.</li><li><code>druid.coordinator.kill.supervisor.durationToRetain</code>: Defines the retention period in <a href="https://en.wikipedia.org/wiki/ISO_8601#Durations" target="_blank" rel="noopener noreferrer">ISO 8601 format</a> after creation that supervisor records become eligible for deletion.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="rules-records">Rules records<a href="#rules-records" class="hash-link" aria-label="Direct link to Rules records" title="Direct link to Rules records"></a></h3><p>Rule records become eligible for deletion when all segments for the datasource have been killed by the kill task and the <code>durationToRetain</code> time has passed since their creation. Automated cleanup for rules requires a <a href="#kill-task">kill task</a>.</p><p>Rule cleanup uses the following configuration:</p><ul><li><code>druid.coordinator.kill.rule.on</code>: When <code>true</code>, enables cleanup for rules records.</li><li><code>druid.coordinator.kill.rule.period</code>: Defines the frequency in <a href="https://en.wikipedia.org/wiki/ISO_8601#Durations" target="_blank" rel="noopener noreferrer">ISO 8601 format</a> for the cleanup job to check for and delete eligible rules records. Defaults to <code>P1D</code>.</li><li><code>druid.coordinator.kill.rule.durationToRetain</code>: Defines the retention period in <a href="https://en.wikipedia.org/wiki/ISO_8601#Durations" target="_blank" rel="noopener noreferrer">ISO 8601 format</a> after creation that rules records become eligible for deletion.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="compaction-configuration-records">Compaction configuration records<a href="#compaction-configuration-records" class="hash-link" aria-label="Direct link to Compaction configuration records" title="Direct link to Compaction configuration records"></a></h3><p>Druid retains all compaction configuration records by default, which should be suitable for most use cases. |
| If you create and delete short-lived datasources with high frequency, and you set auto compaction configuration on those datasources, then consider turning on automated cleanup of compaction configuration records.</p><div class="theme-admonition theme-admonition-info alert alert--info admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg></span>info</div><div class="admonitionContent_S0QG"><p> With automated cleanup of compaction configuration records, if you create a compaction configuration for some datasource before the datasource exists, for example if initial ingestion is still ongoing, Druid may remove the compaction configuration. |
| To prevent the configuration from being prematurely removed, wait for the datasource to be created before applying the compaction configuration to the datasource.</p></div></div><p>Unlike other metadata records, compaction configuration records do not have a retention period set by <code>durationToRetain</code>. Druid deletes compaction configuration records at every cleanup cycle for inactive datasources, which do not have segments either used or unused.</p><p>Compaction configuration records in the <code>druid_config</code> table become eligible for deletion after all segments for the datasource have been killed by the kill task. Automated cleanup for compaction configuration requires a <a href="#kill-task">kill task</a>.</p><p>Compaction configuration cleanup uses the following configuration:</p><ul><li><code>druid.coordinator.kill.compaction.on</code>: When <code>true</code>, enables cleanup for compaction configuration records.</li><li><code>druid.coordinator.kill.compaction.period</code>: Defines the frequency in <a href="https://en.wikipedia.org/wiki/ISO_8601#Durations" target="_blank" rel="noopener noreferrer">ISO 8601 format</a> for the cleanup job to check for and delete eligible compaction configuration records. Defaults to <code>P1D</code>.</li></ul><div class="theme-admonition theme-admonition-info alert alert--info admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg></span>info</div><div class="admonitionContent_S0QG"><p>If you already have an extremely large compaction configuration, you may not be able to delete compaction configuration due to size limits with the audit log. In this case you can set <code>druid.audit.manager.maxPayloadSizeBytes</code> and <code>druid.audit.manager.skipNullField</code> to avoid the auditing issue. See <a href="/docs/latest/configuration/#audit-logging">Audit logging</a>.</p></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="datasource-records-created-by-supervisors">Datasource records created by supervisors<a href="#datasource-records-created-by-supervisors" class="hash-link" aria-label="Direct link to Datasource records created by supervisors" title="Direct link to Datasource records created by supervisors"></a></h3><p>Datasource records created by supervisors become eligible for deletion when the supervisor is terminated or does not exist in the <code>druid_supervisors</code> table and the <code>durationToRetain</code> time has passed since their creation.</p><p>Datasource cleanup uses the following configuration:</p><ul><li><code>druid.coordinator.kill.datasource.on</code>: When <code>true</code>, enables cleanup datasources created by supervisors.</li><li><code>druid.coordinator.kill.datasource.period</code>: Defines the frequency in <a href="https://en.wikipedia.org/wiki/ISO_8601#Durations" target="_blank" rel="noopener noreferrer">ISO 8601 format</a> for the cleanup job to check for and delete eligible datasource records. Defaults to <code>P1D</code>.</li><li><code>druid.coordinator.kill.datasource.durationToRetain</code>: Defines the retention period in <a href="https://en.wikipedia.org/wiki/ISO_8601#Durations" target="_blank" rel="noopener noreferrer">ISO 8601 format</a> after creation that datasource records become eligible for deletion.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="indexer-task-logs">Indexer task logs<a href="#indexer-task-logs" class="hash-link" aria-label="Direct link to Indexer task logs" title="Direct link to Indexer task logs"></a></h3><p>You can configure the Overlord to periodically delete indexer task logs and associated metadata. During cleanup, the Overlord removes the following:</p><ul><li>Indexer task logs from deep storage.</li><li>Indexer task log metadata from the tasks and tasklogs tables in <a href="/docs/latest/configuration/#metadata-storage">metadata storage</a> (named <code>druid_tasks</code> and <code>druid_tasklogs</code> by default). Druid no longer uses the tasklogs table, and the table is always empty.</li></ul><p>To configure cleanup of task logs by the Overlord, set the following properties in the <code>overlord/runtime.properties</code> file.</p><p>Indexer task log cleanup on the Overlord uses the following configuration:</p><ul><li><code>druid.indexer.logs.kill.enabled</code>: When <code>true</code>, enables cleanup of task logs.</li><li><code>druid.indexer.logs.kill.durationToRetain</code>: Defines the length of time in milliseconds to retain task logs.</li><li><code>druid.indexer.logs.kill.initialDelay</code>: Defines the length of time in milliseconds after the Overlord starts before it executes its first job to kill task logs.</li><li><code>druid.indexer.logs.kill.delay</code>: The length of time in milliseconds between jobs to kill task logs.</li></ul><p>For more detail, see <a href="/docs/latest/configuration/#task-logging">Task logging</a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="disable-automated-metadata-cleanup">Disable automated metadata cleanup<a href="#disable-automated-metadata-cleanup" class="hash-link" aria-label="Direct link to Disable automated metadata cleanup" title="Direct link to Disable automated metadata cleanup"></a></h2><p>Druid automatically cleans up metadata records, excluding compaction configuration records and indexer task logs. |
| To disable automated metadata cleanup, set the following properties in the <code>coordinator/runtime.properties</code> file:</p><div class="language-properties codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-properties codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain"># Keep unused segments</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.coordinator.kill.on=false</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"># Keep audit records</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.coordinator.kill.audit.on=false</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"># Keep supervisor records</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.coordinator.kill.supervisor.on=false</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"># Keep rules records</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.coordinator.kill.rule.on=false</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"># Keep datasource records created by supervisors</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.coordinator.kill.datasource.on=false</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><a name="example"></a>## Example configuration for automated metadata cleanup<p>Consider a scenario where you have scripts to create and delete hundreds of datasources and related entities a day. You do not want to fill your metadata store with leftover records. The datasources and related entities tend to persist for only one or two days. Therefore, you want to run a cleanup job that identifies and removes leftover records that are at least four days old. The exception is for audit logs, which you need to retain for 30 days:</p><div class="language-properties codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-properties codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"># Schedule the metadata management store task for every hour:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.coordinator.period.metadataStoreManagementPeriod=P1H</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"># Set a kill task to poll every day to delete Segment records and segments</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"># in deep storage > 4 days old. When druid.coordinator.kill.on is set to true,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"># you can set killDataSourceWhitelist in the dynamic configuration to limit</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"># the datasources that can be killed.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"># Required also for automated cleanup of rules and compaction configuration.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.coordinator.kill.on=true</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.coordinator.kill.period=P1D</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.coordinator.kill.durationToRetain=P4D</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.coordinator.kill.maxSegments=1000</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"># Poll every day to delete audit records > 30 days old</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.coordinator.kill.audit.on=true</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.coordinator.kill.audit.period=P1D</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.coordinator.kill.audit.durationToRetain=P30D</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"># Poll every day to delete supervisor records > 4 days old</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.coordinator.kill.supervisor.on=true</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.coordinator.kill.supervisor.period=P1D</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.coordinator.kill.supervisor.durationToRetain=P4D</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"># Poll every day to delete rules records > 4 days old</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.coordinator.kill.rule.on=true</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.coordinator.kill.rule.period=P1D</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.coordinator.kill.rule.durationToRetain=P4D</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"># Poll every day to delete compaction configuration records</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.coordinator.kill.compaction.on=true</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.coordinator.kill.compaction.period=P1D</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"># Poll every day to delete datasource records created by supervisors > 4 days old</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.coordinator.kill.datasource.on=true</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.coordinator.kill.datasource.period=P1D</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.coordinator.kill.datasource.durationToRetain=P4D</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">...</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="learn-more">Learn more<a href="#learn-more" class="hash-link" aria-label="Direct link to Learn more" title="Direct link to Learn more"></a></h2><p>See the following topics for more information:</p><ul><li><a href="/docs/latest/configuration/#metadata-management">Metadata management</a> for metadata store configuration reference.</li><li><a href="/docs/latest/design/metadata-storage">Metadata storage</a> for an overview of the metadata storage database.</li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/docs/latest/operations/http-compression"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">HTTP compression</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/docs/latest/operations/request-logging"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Request logging</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#automated-cleanup-strategies" class="table-of-contents__link toc-highlight">Automated cleanup strategies</a></li><li><a href="#configure-automated-metadata-cleanup" class="table-of-contents__link toc-highlight">Configure automated metadata cleanup</a><ul><li><a href="#segment-records-and-segments-in-deep-storage-kill-task" class="table-of-contents__link toc-highlight">Segment records and segments in deep storage (kill task)</a></li><li><a href="#audit-records" class="table-of-contents__link toc-highlight">Audit records</a></li><li><a href="#supervisor-records" class="table-of-contents__link toc-highlight">Supervisor records</a></li><li><a href="#rules-records" class="table-of-contents__link toc-highlight">Rules records</a></li><li><a href="#compaction-configuration-records" class="table-of-contents__link toc-highlight">Compaction configuration records</a></li><li><a href="#datasource-records-created-by-supervisors" class="table-of-contents__link toc-highlight">Datasource records created by supervisors</a></li><li><a href="#indexer-task-logs" class="table-of-contents__link toc-highlight">Indexer task logs</a></li></ul></li><li><a href="#disable-automated-metadata-cleanup" class="table-of-contents__link toc-highlight">Disable automated metadata cleanup</a></li><li><a href="#learn-more" class="table-of-contents__link toc-highlight">Learn more</a></li></ul></div></div></div></div></main></div></div><footer class="footer"><div class="container container-fluid"><div class="footer__bottom text--center"><div class="margin-bottom--sm"><img src="/img/favicon.png" class="themedImage_ToTc themedImage--light_HNdA footer__logo"><img src="/img/favicon.png" class="themedImage_ToTc themedImage--dark_i4oU footer__logo"></div><div class="footer__copyright">Copyright © 2023 Apache Software Foundation. Except where otherwise noted, licensed under CC BY-SA 4.0. Apache Druid, Druid, and the Druid logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</div></div></div></footer></div> |
| <script src="/assets/js/runtime~main.26d714fb.js"></script> |
| <script src="/assets/js/main.bd54ee66.js"></script> |
| </body> |
| </html> |
