| <!doctype html> |
| <html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-current plugin-docs plugin-id-default docs-doc-id-design/coordinator" data-has-hydrated="false"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="generator" content="Docusaurus v2.4.3"> |
| <title data-rh="true">Coordinator service | Apache® Druid</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:image" content="https://druid.apache.org/img/druid_nav.png"><meta data-rh="true" name="twitter:image" content="https://druid.apache.org/img/druid_nav.png"><meta data-rh="true" property="og:url" content="https://druid.apache.org/docs/32.0.0/design/coordinator"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="current"><meta data-rh="true" name="docusaurus_tag" content="docs-default-current"><meta data-rh="true" name="docsearch:version" content="current"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-current"><meta data-rh="true" property="og:title" content="Coordinator service | Apache® Druid"><meta data-rh="true" name="description" content="<!--"><meta data-rh="true" property="og:description" content="<!--"><link data-rh="true" rel="icon" href="/img/favicon.png"><link data-rh="true" rel="canonical" href="https://druid.apache.org/docs/32.0.0/design/coordinator"><link data-rh="true" rel="alternate" href="https://druid.apache.org/docs/32.0.0/design/coordinator" hreflang="en"><link data-rh="true" rel="alternate" href="https://druid.apache.org/docs/32.0.0/design/coordinator" hreflang="x-default"><link rel="preconnect" href="https://www.google-analytics.com"> |
| <link rel="preconnect" href="https://www.googletagmanager.com"> |
| <script async src="https://www.googletagmanager.com/gtag/js?id=UA-131010415-1"></script> |
| <script>function gtag(){dataLayer.push(arguments)}window.dataLayer=window.dataLayer||[],gtag("js",new Date),gtag("config","UA-131010415-1",{})</script> |
| |
| |
| |
| |
| |
| <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"> |
| <script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><link rel="stylesheet" href="/assets/css/styles.60a7f877.css"> |
| <link rel="preload" href="/assets/js/runtime~main.0b3aca64.js" as="script"> |
| <link rel="preload" href="/assets/js/main.06dcc5e2.js" as="script"> |
| </head> |
| <body class="navigation-with-keyboard"> |
| <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus"> |
| <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><nav aria-label="Main" class="navbar navbar--fixed-top navbar--dark"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/druid_nav.png" alt="Apache® Druid" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/druid_nav.png" alt="Apache® Druid" class="themedImage_ToTc themedImage--dark_i4oU"></div></a></div><div class="navbar__items navbar__items--right"><a class="navbar__item navbar__link" href="/technology">Technology</a><a class="navbar__item navbar__link" href="/use-cases">Use Cases</a><a class="navbar__item navbar__link" href="/druid-powered">Powered By</a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/docs/32.0.0/design/">Docs</a><a class="navbar__item navbar__link" href="/community/">Community</a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Apache®</a><ul class="dropdown__menu"><li><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Foundation<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://apachecon.com/?ref=druid.apache.org" target="_blank" rel="noopener noreferrer" class="dropdown__link">Events<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/licenses/" target="_blank" rel="noopener noreferrer" class="dropdown__link">License<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Thanks<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/security/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Security<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Sponsorship<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a class="navbar__item navbar__link" href="/downloads/">Download</a><div class="searchBox_ZlJk"><div class="navbar__search"><span aria-label="expand searchbar" role="button" class="search-icon" tabindex="0"></span><input type="search" id="search_input_react" placeholder="Loading..." aria-label="Search" class="navbar__search-input search-bar" disabled=""></div></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/32.0.0/design/">Introduction to Apache Druid</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/32.0.0/tutorials/">Getting started</a><button aria-label="Toggle the collapsible sidebar category 'Getting started'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/docs/32.0.0/design/architecture">Design</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/32.0.0/design/architecture">Architecture</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/docs/32.0.0/design/coordinator">Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/docs/32.0.0/design/coordinator">Coordinator</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/32.0.0/design/overlord">Overlord</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/32.0.0/design/broker">Broker</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/32.0.0/design/router">Router</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/32.0.0/design/historical">Historical</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/32.0.0/design/middlemanager">Middle Manager</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/32.0.0/design/peons">Peon</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/32.0.0/design/indexer">Indexer</a></li></ul></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/32.0.0/design/storage">Storage</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/32.0.0/design/segments">Segments</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/32.0.0/design/deep-storage">Deep storage</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/32.0.0/design/metadata-storage">Metadata storage</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/32.0.0/design/zookeeper">ZooKeeper</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/32.0.0/ingestion/">Ingestion</a><button aria-label="Toggle the collapsible sidebar category 'Ingestion'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/32.0.0/data-management/">Data management</a><button aria-label="Toggle the collapsible sidebar category 'Data management'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/32.0.0/querying/sql">Querying</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/32.0.0/api-reference/">API reference</a><button aria-label="Toggle the collapsible sidebar category 'API reference'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/32.0.0/configuration/">Configuration</a><button aria-label="Toggle the collapsible sidebar category 'Configuration'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/32.0.0/api-reference/automatic-compaction-api">Operations</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/32.0.0/development/overview">Development</a><button aria-label="Toggle the collapsible sidebar category 'Development'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/32.0.0/release-info/release-notes">Release info</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/32.0.0/misc/papers-and-talks">Papers</a></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Design</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Coordinator</span><meta itemprop="position" content="3"></li></ul></nav><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Coordinator service</h1></header><p>The Coordinator service is primarily responsible for segment management and distribution. More specifically, the |
| Coordinator service communicates to Historical services to load or drop segments based on configurations. The Coordinator is responsible for loading new segments, dropping outdated segments, ensuring that segments are "replicated" (that is, loaded on multiple different Historical nodes) proper (configured) number of times, and moving |
| ("balancing") segments between Historical nodes to keep the latter evenly loaded.</p><p>The Coordinator runs its duties periodically and the time between each run is a configurable parameter. On each |
| run, the Coordinator assesses the current state of the cluster before deciding on the appropriate actions to take. |
| Similar to the Broker and Historical services, the Coordinator maintains a connection to a ZooKeeper cluster for |
| current cluster information. The Coordinator also maintains a connection to a database containing information about |
| "used" segments (that is, the segments that <em>should</em> be loaded in the cluster) and the loading rules.</p><p>Before any unassigned segments are serviced by Historical services, the Historical services for each tier are first |
| sorted in terms of capacity, with least capacity servers having the highest priority. Unassigned segments are always |
| assigned to the services with least capacity to maintain a level of balance between services. The Coordinator does not |
| directly communicate with a Historical service when assigning it a new segment; instead the Coordinator creates some |
| temporary information about the new segment under load queue path of the Historical service. Once this request is seen, |
| the Historical service loads the segment and begins servicing it.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="configuration">Configuration<a href="#configuration" class="hash-link" aria-label="Direct link to Configuration" title="Direct link to Configuration"></a></h2><p>For Apache Druid Coordinator service configuration, see <a href="/docs/32.0.0/configuration/#coordinator">Coordinator configuration</a>.</p><p>For basic tuning guidance for the Coordinator service, see <a href="/docs/32.0.0/operations/basic-cluster-tuning#coordinator">Basic cluster tuning</a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="http-endpoints">HTTP endpoints<a href="#http-endpoints" class="hash-link" aria-label="Direct link to HTTP endpoints" title="Direct link to HTTP endpoints"></a></h2><p>For a list of API endpoints supported by the Coordinator, see <a href="/docs/32.0.0/api-reference/service-status-api#coordinator">Service status API reference</a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="running">Running<a href="#running" class="hash-link" aria-label="Direct link to Running" title="Direct link to Running"></a></h2><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">org.apache.druid.cli.Main server coordinator</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="rules">Rules<a href="#rules" class="hash-link" aria-label="Direct link to Rules" title="Direct link to Rules"></a></h2><p>Segments can be automatically loaded and dropped from the cluster based on a set of rules. For more information on rules, see <a href="/docs/32.0.0/operations/rule-configuration">Rule Configuration</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="clean-up-overshadowed-segments">Clean up overshadowed segments<a href="#clean-up-overshadowed-segments" class="hash-link" aria-label="Direct link to Clean up overshadowed segments" title="Direct link to Clean up overshadowed segments"></a></h3><p>On each run, the Coordinator compares the set of used segments in the database with the segments served by some |
| Historical nodes in the cluster. The Coordinator sends requests to Historical nodes to unload unused segments or segments |
| that are removed from the database.</p><p>Segments that are overshadowed (their versions are too old and their data has been replaced by newer segments) are |
| marked as unused. During the next Coordinator's run, they will be unloaded from Historical nodes in the cluster.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="clean-up-non-overshadowed-eternity-tombstone-segments">Clean up non-overshadowed eternity tombstone segments<a href="#clean-up-non-overshadowed-eternity-tombstone-segments" class="hash-link" aria-label="Direct link to Clean up non-overshadowed eternity tombstone segments" title="Direct link to Clean up non-overshadowed eternity tombstone segments"></a></h3><p>On each run, the Coordinator determines and cleans up unneeded eternity tombstone segments for each datasource. These segments must fit all the following criteria:</p><ul><li>It is a tombstone segment that starts at -INF or ends at INF (for example, a tombstone with an interval of <code>-146136543-09-08T08:23:32.096Z/2000-01-01</code> or <code>2020-01-01/146140482-04-24T15:36:27.903Z</code> or <code>-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z</code>)</li><li>It does not overlap with any overshadowed segment</li><li>It has 0 core partitions</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="segment-availability">Segment availability<a href="#segment-availability" class="hash-link" aria-label="Direct link to Segment availability" title="Direct link to Segment availability"></a></h2><p>If a Historical service restarts or becomes unavailable for any reason, the Coordinator notices that a service has gone missing and treats all segments served by that service as being dropped. The segments are then reassigned to other Historical services in the cluster. However, each segment that is dropped is not immediately forgotten. Instead, there is a transitional data structure that stores all dropped segments with an associated lifetime. The lifetime represents a period of time in which the Coordinator will not reassign a dropped segment. Hence, if a Historical service becomes unavailable and available again within a short period of time, the Historical service will start up and serve segments from its cache without any of those segments being reassigned across the cluster.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="balancing-segments-in-a-tier">Balancing segments in a tier<a href="#balancing-segments-in-a-tier" class="hash-link" aria-label="Direct link to Balancing segments in a tier" title="Direct link to Balancing segments in a tier"></a></h2><p>Druid queries perform optimally when segments are distributed evenly across Historical services. An ideal distribution would ensure that all Historicals participate equally in the query load thus avoiding hot-spots in the system. To some extent, this can be achieved by keeping multiple replicas of a segment in a cluster. |
| But in a tier with several Historicals (or a low replication factor), segment replication is not sufficient to attain balance. |
| Thus, the Coordinator constantly monitors the set of segments present on each Historical in a tier and employs one of the following strategies to identify segments that may be moved from one Historical to another to retain balance.</p><ul><li><code>cost</code> (default): For a given segment in a tier, this strategy picks the server with the minimum "cost" of placing that segment. The cost is a function of the data interval of the segment and the data intervals of all the segments already present on the candidate server. In essence, this strategy tries to avoid placing segments with adjacent or overlapping data intervals on the same server. This is based on the premise that adjacent-interval segments are more likely to be used together in a query and placing them on the same server may lead to skewed CPU usages of Historicals.</li><li><code>diskNormalized</code>: A derivative of the <code>cost</code> strategy that weights the cost of placing a segment on a server with the disk usage ratio of the server. There are known issues with this strategy and is not recommended for a production cluster.</li><li><code>random</code>: Distributes segments randomly across servers. This is an experimental strategy and is not recommended for a production cluster.</li></ul><p>All of the above strategies prioritize moving segments from the Historical with the least available disk space.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="automatic-compaction">Automatic compaction<a href="#automatic-compaction" class="hash-link" aria-label="Direct link to Automatic compaction" title="Direct link to Automatic compaction"></a></h2><p>The Coordinator manages the <a href="/docs/32.0.0/data-management/automatic-compaction">automatic compaction system</a>. |
| Each run, the Coordinator compacts segments by merging small segments or splitting a large one. This is useful when the size of your segments is not optimized which may degrade query performance. |
| See <a href="/docs/32.0.0/operations/segment-optimization">Segment size optimization</a> for details.</p><p>The Coordinator first finds the segments to compact based on the <a href="#segment-search-policy-in-automatic-compaction">segment search policy</a>. |
| Once some segments are found, it issues a <a href="/docs/32.0.0/ingestion/tasks#compact">compaction task</a> to compact those segments. |
| The maximum number of running compaction tasks is <code>min(sum of worker capacity * slotRatio, maxSlots)</code>. |
| Note that even if <code>min(sum of worker capacity * slotRatio, maxSlots) = 0</code>, at least one compaction task is always submitted |
| if the compaction is enabled for a dataSource. |
| See <a href="/docs/32.0.0/api-reference/automatic-compaction-api#manage-automatic-compaction">Automatic compaction configuration API</a> and <a href="/docs/32.0.0/configuration/#automatic-compaction-dynamic-configuration">Automatic compaction configuration</a> to enable and configure automatic compaction.</p><p>Compaction tasks might fail due to the following reasons:</p><ul><li>If the input segments of a compaction task are removed or overshadowed before it starts, that compaction task fails immediately.</li><li>If a task of a higher priority acquires a <a href="/docs/32.0.0/ingestion/tasks#locking">time chunk lock</a> for an interval overlapping with the interval of a compaction task, the compaction task fails.</li></ul><p>Once a compaction task fails, the Coordinator simply checks the segments in the interval of the failed task again, and issues another compaction task in the next run.</p><p>Note that Compacting Segments Coordinator Duty is automatically enabled and run as part of the Indexing Service Duties group. However, Compacting Segments Coordinator Duty can be configured to run in isolation as a separate Coordinator duty group. This allows changing the period of Compacting Segments Coordinator Duty without impacting the period of other Indexing Service Duties. This can be done by setting the following properties. For more details, see <a href="/docs/32.0.0/development/modules#adding-your-own-custom-pluggable-coordinator-duty">custom pluggable Coordinator Duty</a>.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.coordinator.dutyGroups=[<SOME_GROUP_NAME>]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.coordinator.<SOME_GROUP_NAME>.duties=["compactSegments"]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.coordinator.<SOME_GROUP_NAME>.period=<PERIOD_TO_RUN_COMPACTING_SEGMENTS_DUTY></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="segment-search-policy-in-automatic-compaction">Segment search policy in automatic compaction<a href="#segment-search-policy-in-automatic-compaction" class="hash-link" aria-label="Direct link to Segment search policy in automatic compaction" title="Direct link to Segment search policy in automatic compaction"></a></h2><p>At every Coordinator run, this policy looks up time chunks from newest to oldest and checks whether the segments in those time chunks |
| need compaction. |
| A set of segments needs compaction if all conditions below are satisfied:</p><ul><li>Total size of segments in the time chunk is smaller than or equal to the configured <code>inputSegmentSizeBytes</code>.</li><li>Segments have never been compacted yet or compaction spec has been updated since the last compaction: <code>maxTotalRows</code> or <code>indexSpec</code>.</li></ul><p>Here are some details with an example. Suppose we have two dataSources (<code>foo</code>, <code>bar</code>) as seen below:</p><ul><li><code>foo</code><ul><li><code>foo_2017-11-01T00:00:00.000Z_2017-12-01T00:00:00.000Z_VERSION</code></li><li><code>foo_2017-11-01T00:00:00.000Z_2017-12-01T00:00:00.000Z_VERSION_1</code></li><li><code>foo_2017-09-01T00:00:00.000Z_2017-10-01T00:00:00.000Z_VERSION</code></li></ul></li><li><code>bar</code><ul><li><code>bar_2017-10-01T00:00:00.000Z_2017-11-01T00:00:00.000Z_VERSION</code></li><li><code>bar_2017-10-01T00:00:00.000Z_2017-11-01T00:00:00.000Z_VERSION_1</code></li></ul></li></ul><p>Assuming that each segment is 10 MB and haven't been compacted yet, this policy first returns two segments of |
| <code>foo_2017-11-01T00:00:00.000Z_2017-12-01T00:00:00.000Z_VERSION</code> and <code>foo_2017-11-01T00:00:00.000Z_2017-12-01T00:00:00.000Z_VERSION_1</code> to compact together because |
| <code>2017-11-01T00:00:00.000Z/2017-12-01T00:00:00.000Z</code> is the most recent time chunk.</p><p>If the Coordinator has enough task slots for compaction, this policy will continue searching for the next segments and return |
| <code>bar_2017-10-01T00:00:00.000Z_2017-11-01T00:00:00.000Z_VERSION</code> and <code>bar_2017-10-01T00:00:00.000Z_2017-11-01T00:00:00.000Z_VERSION_1</code>. |
| Finally, <code>foo_2017-09-01T00:00:00.000Z_2017-10-01T00:00:00.000Z_VERSION</code> will be picked up even though there is only one segment in the time chunk of <code>2017-09-01T00:00:00.000Z/2017-10-01T00:00:00.000Z</code>.</p><p>The search start point can be changed by setting <code>skipOffsetFromLatest</code>. |
| If this is set, this policy will ignore the segments falling into the time chunk of (the end time of the most recent segment - <code>skipOffsetFromLatest</code>). |
| This is to avoid conflicts between compaction tasks and realtime tasks. |
| Note that realtime tasks have a higher priority than compaction tasks by default. Realtime tasks will revoke the locks of compaction tasks if their intervals overlap, resulting in the termination of the compaction task. |
| For more information, see <a href="/docs/32.0.0/data-management/automatic-compaction#avoid-conflicts-with-ingestion">Avoid conflicts with ingestion</a>.</p><div class="theme-admonition theme-admonition-info alert alert--info admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg></span>info</div><div class="admonitionContent_S0QG"><p> This policy currently cannot handle the situation when there are a lot of small segments which have the same interval, |
| and their total size exceeds <a href="/docs/32.0.0/configuration/#automatic-compaction-dynamic-configuration"><code>inputSegmentSizeBytes</code></a>. |
| If it finds such segments, it simply skips them.</p></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="faq">FAQ<a href="#faq" class="hash-link" aria-label="Direct link to FAQ" title="Direct link to FAQ"></a></h2><ol><li><p><strong>Do clients ever contact the Coordinator service?</strong></p><p>The Coordinator is not involved in a query.</p><p>Historical services never directly contact the Coordinator service. The Coordinator tells the Historical services to load/drop data via ZooKeeper, but the Historical services are completely unaware of the Coordinator.</p><p>Brokers also never contact the Coordinator. Brokers base their understanding of the data topology on metadata exposed by the Historical services via ZooKeeper and are completely unaware of the Coordinator.</p></li><li><p><strong>Does it matter if the Coordinator service starts up before or after other services?</strong></p><p>No. If the Coordinator is not started up, no new segments will be loaded in the cluster and outdated segments will not be dropped. However, the Coordinator service can be started up at any time, and after a configurable delay, will start running Coordinator tasks.</p><p>This also means that if you have a working cluster and all of your Coordinators die, the cluster will continue to function, it just won’t experience any changes to its data topology.</p></li></ol></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/docs/32.0.0/design/architecture"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Architecture</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/docs/32.0.0/design/overlord"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Overlord</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#configuration" class="table-of-contents__link toc-highlight">Configuration</a></li><li><a href="#http-endpoints" class="table-of-contents__link toc-highlight">HTTP endpoints</a></li><li><a href="#running" class="table-of-contents__link toc-highlight">Running</a></li><li><a href="#rules" class="table-of-contents__link toc-highlight">Rules</a><ul><li><a href="#clean-up-overshadowed-segments" class="table-of-contents__link toc-highlight">Clean up overshadowed segments</a></li><li><a href="#clean-up-non-overshadowed-eternity-tombstone-segments" class="table-of-contents__link toc-highlight">Clean up non-overshadowed eternity tombstone segments</a></li></ul></li><li><a href="#segment-availability" class="table-of-contents__link toc-highlight">Segment availability</a></li><li><a href="#balancing-segments-in-a-tier" class="table-of-contents__link toc-highlight">Balancing segments in a tier</a></li><li><a href="#automatic-compaction" class="table-of-contents__link toc-highlight">Automatic compaction</a></li><li><a href="#segment-search-policy-in-automatic-compaction" class="table-of-contents__link toc-highlight">Segment search policy in automatic compaction</a></li><li><a href="#faq" class="table-of-contents__link toc-highlight">FAQ</a></li></ul></div></div></div></div></main></div></div><footer class="footer"><div class="container container-fluid"><div class="footer__bottom text--center"><div class="margin-bottom--sm"><img src="/img/favicon.png" class="themedImage_ToTc themedImage--light_HNdA footer__logo"><img src="/img/favicon.png" class="themedImage_ToTc themedImage--dark_i4oU footer__logo"></div><div class="footer__copyright">Copyright © 2023 Apache Software Foundation. Except where otherwise noted, licensed under CC BY-SA 4.0. Apache Druid, Druid, and the Druid logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</div></div></div></footer></div> |
| <script src="/assets/js/runtime~main.0b3aca64.js"></script> |
| <script src="/assets/js/main.06dcc5e2.js"></script> |
| </body> |
| </html> |
