| <!doctype html> |
| <html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-current plugin-docs plugin-id-default docs-doc-id-operations/basic-cluster-tuning"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="generator" content="Docusaurus v2.4.1"> |
| <title data-rh="true">Basic cluster tuning | Apache® Druid</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:image" content="https://druid.apache.org/img/druid_nav.png"><meta data-rh="true" name="twitter:image" content="https://druid.apache.org/img/druid_nav.png"><meta data-rh="true" property="og:url" content="https://druid.apache.org/docs/26.0.0/operations/basic-cluster-tuning"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="current"><meta data-rh="true" name="docusaurus_tag" content="docs-default-current"><meta data-rh="true" name="docsearch:version" content="current"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-current"><meta data-rh="true" property="og:title" content="Basic cluster tuning | Apache® Druid"><meta data-rh="true" name="description" content="<!--"><meta data-rh="true" property="og:description" content="<!--"><link data-rh="true" rel="icon" href="/img/favicon.png"><link data-rh="true" rel="canonical" href="https://druid.apache.org/docs/26.0.0/operations/basic-cluster-tuning"><link data-rh="true" rel="alternate" href="https://druid.apache.org/docs/26.0.0/operations/basic-cluster-tuning" hreflang="en"><link data-rh="true" rel="alternate" href="https://druid.apache.org/docs/26.0.0/operations/basic-cluster-tuning" hreflang="x-default"><link rel="preconnect" href="https://www.google-analytics.com"> |
| <link rel="preconnect" href="https://www.googletagmanager.com"> |
| <script async src="https://www.googletagmanager.com/gtag/js?id=UA-131010415-1"></script> |
| <script>function gtag(){dataLayer.push(arguments)}window.dataLayer=window.dataLayer||[],gtag("js",new Date),gtag("config","UA-131010415-1",{})</script> |
| |
| |
| |
| |
| <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"> |
| <script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><link rel="stylesheet" href="/assets/css/styles.f80751b3.css"> |
| <link rel="preload" href="/assets/js/runtime~main.38900cbf.js" as="script"> |
| <link rel="preload" href="/assets/js/main.5e106d68.js" as="script"> |
| </head> |
| <body class="navigation-with-keyboard"> |
| <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus"> |
| <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><nav aria-label="Main" class="navbar navbar--fixed-top navbar--dark"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/druid_nav.png" alt="Apache® Druid" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/druid_nav.png" alt="Apache® Druid" class="themedImage_ToTc themedImage--dark_i4oU"></div></a></div><div class="navbar__items navbar__items--right"><a class="navbar__item navbar__link" href="/technology">Technology</a><a class="navbar__item navbar__link" href="/use-cases">Use Cases</a><a class="navbar__item navbar__link" href="/druid-powered">Powered By</a><a class="navbar__item navbar__link" href="/docs/26.0.0/design/">Docs</a><a class="navbar__item navbar__link" href="/community/">Community</a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Apache®</a><ul class="dropdown__menu"><li><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Foundation<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://apachecon.com/?ref=druid.apache.org" target="_blank" rel="noopener noreferrer" class="dropdown__link">Events<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/licenses/" target="_blank" rel="noopener noreferrer" class="dropdown__link">License<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Thanks<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/security/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Security<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Sponsorship<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a class="navbar__item navbar__link" href="/downloads/">Download</a><div class="searchBox_ZlJk"><div class="navbar__search"><span aria-label="expand searchbar" role="button" class="search-icon" tabindex="0"></span><input type="search" id="search_input_react" placeholder="Loading..." aria-label="Search" class="navbar__search-input search-bar" disabled=""></div></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/26.0.0/design/">Getting started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/26.0.0/tutorials/tutorial-batch">Tutorials</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/26.0.0/design/architecture">Design</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/26.0.0/ingestion/">Ingestion</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/26.0.0/data-management/">Data management</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/26.0.0/querying/sql">Querying</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/26.0.0/configuration/">Configuration</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/docs/26.0.0/operations/web-console">Operations</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/operations/web-console">Web console</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/operations/java">Java runtime</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/26.0.0/operations/security-overview">Security</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/docs/26.0.0/operations/basic-cluster-tuning">Performance tuning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/docs/26.0.0/operations/basic-cluster-tuning">Basic cluster tuning</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/operations/segment-optimization">Segment size optimization</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/operations/mixed-workloads">Mixed workloads</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/operations/http-compression">HTTP compression</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/operations/clean-metadata-store">Automated metadata cleanup</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/26.0.0/operations/request-logging">Monitoring</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/operations/api-reference">API reference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/operations/high-availability">High availability</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/operations/rolling-updates">Rolling updates</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/operations/rule-configuration">Using rules to drop and retain data</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/operations/other-hadoop">Working with different versions of Apache Hadoop</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/26.0.0/operations/dump-segment">Misc</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/26.0.0/development/overview">Development</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/26.0.0/misc/papers-and-talks">Misc</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Operations</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Performance tuning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Basic cluster tuning</span><meta itemprop="position" content="3"></li></ul></nav><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Basic cluster tuning</h1></header><p>This document provides basic guidelines for configuration properties and cluster architecture considerations related to performance tuning of an Apache Druid deployment.</p><p>Please note that this document provides general guidelines and rules-of-thumb: these are not absolute, universal rules for cluster tuning, and this introductory guide is not an exhaustive description of all Druid tuning properties, which are described in the <a href="/docs/26.0.0/configuration/">configuration reference</a>.</p><p>If you have questions on tuning Druid for specific use cases, or questions on configuration properties not covered in this guide, please ask the <a href="https://druid.apache.org/community/" target="_blank" rel="noopener noreferrer">Druid user mailing list or other community channels</a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="process-specific-guidelines">Process-specific guidelines<a href="#process-specific-guidelines" class="hash-link" aria-label="Direct link to Process-specific guidelines" title="Direct link to Process-specific guidelines"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="historical">Historical<a href="#historical" class="hash-link" aria-label="Direct link to Historical" title="Direct link to Historical"></a></h3><h4 class="anchor anchorWithStickyNavbar_LWe7" id="heap-sizing">Heap sizing<a href="#heap-sizing" class="hash-link" aria-label="Direct link to Heap sizing" title="Direct link to Heap sizing"></a></h4><p>The biggest contributions to heap usage on Historicals are:</p><ul><li>Partial unmerged query results from segments</li><li>The stored maps for <a href="/docs/26.0.0/querying/lookups">lookups</a>.</li></ul><p>A general rule-of-thumb for sizing the Historical heap is <code>(0.5GiB * number of CPU cores)</code>, with an upper limit of ~24GiB.</p><p>This rule-of-thumb scales using the number of CPU cores as a convenient proxy for hardware size and level of concurrency (note: this formula is not a hard rule for sizing Historical heaps).</p><p>Having a heap that is too large can result in excessively long GC collection pauses, the ~24GiB upper limit is imposed to avoid this.</p><p>If caching is enabled on Historicals, the cache is stored on heap, sized by <code>druid.cache.sizeInBytes</code>.</p><p>Running out of heap on the Historicals can indicate misconfiguration or usage patterns that are overloading the cluster.</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="lookups">Lookups<a href="#lookups" class="hash-link" aria-label="Direct link to Lookups" title="Direct link to Lookups"></a></h5><p>If you are using lookups, calculate the total size of the lookup maps being loaded.</p><p>Druid performs an atomic swap when updating lookup maps (both the old map and the new map will exist in heap during the swap), so the maximum potential heap usage from lookup maps will be (2 * total size of all loaded lookups).</p><p>Be sure to add <code>(2 * total size of all loaded lookups)</code> to your heap size in addition to the <code>(0.5GiB * number of CPU cores)</code> guideline.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="processing-threads-and-buffers">Processing Threads and Buffers<a href="#processing-threads-and-buffers" class="hash-link" aria-label="Direct link to Processing Threads and Buffers" title="Direct link to Processing Threads and Buffers"></a></h4><p>Please see the <a href="#processing-threads-buffers">General Guidelines for Processing Threads and Buffers</a> section for an overview of processing thread/buffer configuration.</p><p>On Historicals:</p><ul><li><code>druid.processing.numThreads</code> should generally be set to <code>(number of cores - 1)</code>: a smaller value can result in CPU underutilization, while going over the number of cores can result in unnecessary CPU contention.</li><li><code>druid.processing.buffer.sizeBytes</code> can be set to 500MiB.</li><li><code>druid.processing.numMergeBuffers</code>, a 1:4 ratio of merge buffers to processing threads is a reasonable choice for general use.</li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="direct-memory-sizing">Direct Memory Sizing<a href="#direct-memory-sizing" class="hash-link" aria-label="Direct link to Direct Memory Sizing" title="Direct link to Direct Memory Sizing"></a></h4><p>The processing and merge buffers described above are direct memory buffers.</p><p>When a historical processes a query, it must open a set of segments for reading. This also requires some direct memory space, described in <a href="#segment-decompression">segment decompression buffers</a>.</p><p>A formula for estimating direct memory usage follows:</p><p>(<code>druid.processing.numThreads</code> + <code>druid.processing.numMergeBuffers</code> + 1) * <code>druid.processing.buffer.sizeBytes</code></p><p>The <code>+ 1</code> factor is a fuzzy estimate meant to account for the segment decompression buffers.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="connection-pool-sizing">Connection pool sizing<a href="#connection-pool-sizing" class="hash-link" aria-label="Direct link to Connection pool sizing" title="Direct link to Connection pool sizing"></a></h4><p>Please see the <a href="#connection-pool">General Connection Pool Guidelines</a> section for an overview of connection pool configuration.</p><p>For Historicals, <code>druid.server.http.numThreads</code> should be set to a value slightly higher than the sum of <code>druid.broker.http.numConnections</code> across all the Brokers in the cluster.</p><p>Tuning the cluster so that each Historical can accept 50 queries and 10 non-queries is a reasonable starting point.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="segment-cache-size">Segment Cache Size<a href="#segment-cache-size" class="hash-link" aria-label="Direct link to Segment Cache Size" title="Direct link to Segment Cache Size"></a></h4><p>For better query performance, do not allocate segment data to a Historical in excess of the system free memory. When <code>free system memory</code> is greater than or equal to <code>druid.segmentCache.locations</code>, the more segment data the Historical can be held in the memory-mapped segment cache.</p><p>Druid uses the <code>druid.segmentCache.locations</code> to calculate the total segment data size assigned to a Historical. For some rarer use cases, you can override this behavior with <code>druid.server.maxSize</code> property.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="number-of-historicals">Number of Historicals<a href="#number-of-historicals" class="hash-link" aria-label="Direct link to Number of Historicals" title="Direct link to Number of Historicals"></a></h4><p>The number of Historicals needed in a cluster depends on how much data the cluster has. For good performance, you will want enough Historicals such that each Historical has a good (<code>free system memory</code> / total size of all <code>druid.segmentCache.locations</code>) ratio, as described in the segment cache size section above.</p><p>Having a smaller number of big servers is generally better than having a large number of small servers, as long as you have enough fault tolerance for your use case.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="ssd-storage">SSD storage<a href="#ssd-storage" class="hash-link" aria-label="Direct link to SSD storage" title="Direct link to SSD storage"></a></h4><p>We recommend using SSDs for storage on the Historicals, as they handle segment data stored on disk.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="total-memory-usage">Total memory usage<a href="#total-memory-usage" class="hash-link" aria-label="Direct link to Total memory usage" title="Direct link to Total memory usage"></a></h4><p>To estimate total memory usage of the Historical under these guidelines:</p><ul><li>Heap: <code>(0.5GiB * number of CPU cores) + (2 * total size of lookup maps) + druid.cache.sizeInBytes</code></li><li>Direct Memory: <code>(druid.processing.numThreads + druid.processing.numMergeBuffers + 1) * druid.processing.buffer.sizeBytes</code></li></ul><p>The Historical will use any available free system memory (i.e., memory not used by the Historical JVM and heap/direct memory buffers or other processes on the system) for memory-mapping of segments on disk. For better query performance, you will want to ensure a good (<code>free system memory</code> / total size of all <code>druid.segmentCache.locations</code>) ratio so that a greater proportion of segments can be kept in memory.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="segment-sizes-matter">Segment sizes matter<a href="#segment-sizes-matter" class="hash-link" aria-label="Direct link to Segment sizes matter" title="Direct link to Segment sizes matter"></a></h4><p>Be sure to check out <a href="/docs/26.0.0/operations/segment-optimization">segment size optimization</a> to help tune your Historical processes for maximum performance.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="broker">Broker<a href="#broker" class="hash-link" aria-label="Direct link to Broker" title="Direct link to Broker"></a></h3><h4 class="anchor anchorWithStickyNavbar_LWe7" id="heap-sizing-1">Heap sizing<a href="#heap-sizing-1" class="hash-link" aria-label="Direct link to Heap sizing" title="Direct link to Heap sizing"></a></h4><p>The biggest contributions to heap usage on Brokers are:</p><ul><li>Partial unmerged query results from Historicals and Tasks</li><li>The segment timeline: this consists of location information (which Historical/Task is serving a segment) for all currently <a href="/docs/26.0.0/design/architecture#segment-lifecycle">available</a> segments.</li><li>Cached segment metadata: this consists of metadata, such as per-segment schemas, for all currently available segments.</li></ul><p>The Broker heap requirements scale based on the number of segments in the cluster, and the total data size of the segments.</p><p>The heap size will vary based on data size and usage patterns, but 4GiB to 8GiB is a good starting point for a small or medium cluster (~15 servers or less). For a rough estimate of memory requirements on the high end, very large clusters with a node count on the order of ~100 nodes may need Broker heaps of 30GiB-60GiB.</p><p>If caching is enabled on the Broker, the cache is stored on heap, sized by <code>druid.cache.sizeInBytes</code>.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="direct-memory-sizing-1">Direct memory sizing<a href="#direct-memory-sizing-1" class="hash-link" aria-label="Direct link to Direct memory sizing" title="Direct link to Direct memory sizing"></a></h4><p>On the Broker, the amount of direct memory needed depends on how many merge buffers (used for merging GroupBys) are configured. The Broker does not generally need processing threads or processing buffers, as query results are merged on-heap in the HTTP connection threads instead.</p><ul><li><code>druid.processing.buffer.sizeBytes</code> can be set to 500MiB.</li><li><code>druid.processing.numMergeBuffers</code>: set this to the same value as on Historicals or a bit higher</li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="connection-pool-sizing-1">Connection pool sizing<a href="#connection-pool-sizing-1" class="hash-link" aria-label="Direct link to Connection pool sizing" title="Direct link to Connection pool sizing"></a></h4><p>Please see the <a href="#connection-pool">General Connection Pool Guidelines</a> section for an overview of connection pool configuration.</p><p>On the Brokers, please ensure that the sum of <code>druid.broker.http.numConnections</code> across all the Brokers is slightly lower than the value of <code>druid.server.http.numThreads</code> on your Historicals and Tasks.</p><p><code>druid.server.http.numThreads</code> on the Broker should be set to a value slightly higher than <code>druid.broker.http.numConnections</code> on the same Broker.</p><p>Tuning the cluster so that each Historical can accept 50 queries and 10 non-queries, adjusting the Brokers accordingly, is a reasonable starting point.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="broker-backpressure">Broker backpressure<a href="#broker-backpressure" class="hash-link" aria-label="Direct link to Broker backpressure" title="Direct link to Broker backpressure"></a></h4><p>When retrieving query results from Historical processes or Tasks, the Broker can optionally specify a maximum buffer size for queued, unread data, and exert backpressure on the channel to the Historical or Tasks when limit is reached (causing writes to the channel to block on the Historical/Task side until the Broker is able to drain some data from the channel).</p><p>This buffer size is controlled by the <code>druid.broker.http.maxQueuedBytes</code> setting.</p><p>The limit is divided across the number of Historicals/Tasks that a query would hit: suppose I have <code>druid.broker.http.maxQueuedBytes</code> set to 5MiB, and the Broker receives a query that needs to be fanned out to 2 Historicals. Each per-historical channel would get a 2.5MiB buffer in this case.</p><p>You can generally set this to a value of approximately <code>2MiB * number of Historicals</code>. As your cluster scales up with more Historicals and Tasks, consider increasing this buffer size and increasing the Broker heap accordingly.</p><ul><li>If the buffer is too small, this can lead to inefficient queries due to the buffer filling up rapidly and stalling the channel</li><li>If the buffer is too large, this puts more memory pressure on the Broker due to more queued result data in the HTTP channels.</li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="number-of-brokers">Number of brokers<a href="#number-of-brokers" class="hash-link" aria-label="Direct link to Number of brokers" title="Direct link to Number of brokers"></a></h4><p>A 1:15 ratio of Brokers to Historicals is a reasonable starting point (this is not a hard rule).</p><p>If you need Broker HA, you can deploy 2 initially and then use the 1:15 ratio guideline for additional Brokers.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="total-memory-usage-1">Total memory usage<a href="#total-memory-usage-1" class="hash-link" aria-label="Direct link to Total memory usage" title="Direct link to Total memory usage"></a></h4><p>To estimate total memory usage of the Broker under these guidelines:</p><ul><li>Heap: allocated heap size</li><li>Direct Memory: <code>(druid.processing.numMergeBuffers + 1) * druid.processing.buffer.sizeBytes</code></li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="middlemanager">MiddleManager<a href="#middlemanager" class="hash-link" aria-label="Direct link to MiddleManager" title="Direct link to MiddleManager"></a></h3><p>The MiddleManager is a lightweight task controller/manager that launches Task processes, which perform ingestion work.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="middlemanager-heap-sizing">MiddleManager heap sizing<a href="#middlemanager-heap-sizing" class="hash-link" aria-label="Direct link to MiddleManager heap sizing" title="Direct link to MiddleManager heap sizing"></a></h4><p>The MiddleManager itself does not require much resources, you can set the heap to ~128MiB generally.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="ssd-storage-1">SSD storage<a href="#ssd-storage-1" class="hash-link" aria-label="Direct link to SSD storage" title="Direct link to SSD storage"></a></h4><p>We recommend using SSDs for storage on the MiddleManagers, as the Tasks launched by MiddleManagers handle segment data stored on disk.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="task-count">Task Count<a href="#task-count" class="hash-link" aria-label="Direct link to Task Count" title="Direct link to Task Count"></a></h4><p>The number of tasks a MiddleManager can launch is controlled by the <code>druid.worker.capacity</code> setting.</p><p>The number of workers needed in your cluster depends on how many concurrent ingestion tasks you need to run for your use cases. The number of workers that can be launched on a given machine depends on the size of resources allocated per worker and available system resources.</p><p>You can allocate more MiddleManager machines to your cluster to add task capacity.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="task-configurations">Task configurations<a href="#task-configurations" class="hash-link" aria-label="Direct link to Task configurations" title="Direct link to Task configurations"></a></h4><p>The following section below describes configuration for Tasks launched by the MiddleManager. The Tasks can be queried and perform ingestion workloads, so they require more resources than the MM.</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="task-heap-sizing">Task heap sizing<a href="#task-heap-sizing" class="hash-link" aria-label="Direct link to Task heap sizing" title="Direct link to Task heap sizing"></a></h5><p>A 1GiB heap is usually enough for Tasks.</p><h6 class="anchor anchorWithStickyNavbar_LWe7" id="lookups-1">Lookups<a href="#lookups-1" class="hash-link" aria-label="Direct link to Lookups" title="Direct link to Lookups"></a></h6><p>If you are using lookups, calculate the total size of the lookup maps being loaded.</p><p>Druid performs an atomic swap when updating lookup maps (both the old map and the new map will exist in heap during the swap), so the maximum potential heap usage from lookup maps will be (2 * total size of all loaded lookups).</p><p>Be sure to add <code>(2 * total size of all loaded lookups)</code> to your Task heap size if you are using lookups.</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="task-processing-threads-and-buffers">Task processing threads and buffers<a href="#task-processing-threads-and-buffers" class="hash-link" aria-label="Direct link to Task processing threads and buffers" title="Direct link to Task processing threads and buffers"></a></h5><p>For Tasks, 1 or 2 processing threads are often enough, as the Tasks tend to hold much less queryable data than Historical processes.</p><ul><li><code>druid.indexer.fork.property.druid.processing.numThreads</code>: set this to 1 or 2</li><li><code>druid.indexer.fork.property.druid.processing.numMergeBuffers</code>: set this to 2</li><li><code>druid.indexer.fork.property.druid.processing.buffer.sizeBytes</code>: can be set to 100MiB</li></ul><h5 class="anchor anchorWithStickyNavbar_LWe7" id="direct-memory-sizing-2">Direct memory sizing<a href="#direct-memory-sizing-2" class="hash-link" aria-label="Direct link to Direct memory sizing" title="Direct link to Direct memory sizing"></a></h5><p>The processing and merge buffers described above are direct memory buffers.</p><p>When a Task processes a query, it must open a set of segments for reading. This also requires some direct memory space, described in <a href="#segment-decompression">segment decompression buffers</a>.</p><p>An ingestion Task also needs to merge partial ingestion results, which requires direct memory space, described in <a href="#segment-merging">segment merging</a>.</p><p>A formula for estimating direct memory usage follows:</p><p>(<code>druid.processing.numThreads</code> + <code>druid.processing.numMergeBuffers</code> + 1) * <code>druid.processing.buffer.sizeBytes</code></p><p>The <code>+ 1</code> factor is a fuzzy estimate meant to account for the segment decompression buffers and dictionary merging buffers.</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="connection-pool-sizing-2">Connection pool sizing<a href="#connection-pool-sizing-2" class="hash-link" aria-label="Direct link to Connection pool sizing" title="Direct link to Connection pool sizing"></a></h5><p>Please see the <a href="#connection-pool">General Connection Pool Guidelines</a> section for an overview of connection pool configuration.</p><p>For Tasks, <code>druid.server.http.numThreads</code> should be set to a value slightly higher than the sum of <code>druid.broker.http.numConnections</code> across all the Brokers in the cluster.</p><p>Tuning the cluster so that each Task can accept 50 queries and 10 non-queries is a reasonable starting point.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="total-memory-usage-2">Total memory usage<a href="#total-memory-usage-2" class="hash-link" aria-label="Direct link to Total memory usage" title="Direct link to Total memory usage"></a></h4><p>To estimate total memory usage of a Task under these guidelines:</p><ul><li>Heap: <code>1GiB + (2 * total size of lookup maps)</code></li><li>Direct Memory: <code>(druid.processing.numThreads + druid.processing.numMergeBuffers + 1) * druid.processing.buffer.sizeBytes</code></li></ul><p>The total memory usage of the MiddleManager + Tasks:</p><p><code>MM heap size + druid.worker.capacity * (single task memory usage)</code></p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="configuration-guidelines-for-specific-ingestion-types">Configuration guidelines for specific ingestion types<a href="#configuration-guidelines-for-specific-ingestion-types" class="hash-link" aria-label="Direct link to Configuration guidelines for specific ingestion types" title="Direct link to Configuration guidelines for specific ingestion types"></a></h5><h6 class="anchor anchorWithStickyNavbar_LWe7" id="kafkakinesis-ingestion">Kafka/Kinesis ingestion<a href="#kafkakinesis-ingestion" class="hash-link" aria-label="Direct link to Kafka/Kinesis ingestion" title="Direct link to Kafka/Kinesis ingestion"></a></h6><p>If you use the <a href="/docs/26.0.0/development/extensions-core/kafka-ingestion">Kafka Indexing Service</a> or <a href="/docs/26.0.0/development/extensions-core/kinesis-ingestion">Kinesis Indexing Service</a>, the number of tasks required will depend on the number of partitions and your taskCount/replica settings.</p><p>On top of those requirements, allocating more task slots in your cluster is a good idea, so that you have free task |
| slots available for other tasks, such as <a href="/docs/26.0.0/data-management/compaction">compaction tasks</a>.</p><h6 class="anchor anchorWithStickyNavbar_LWe7" id="hadoop-ingestion">Hadoop ingestion<a href="#hadoop-ingestion" class="hash-link" aria-label="Direct link to Hadoop ingestion" title="Direct link to Hadoop ingestion"></a></h6><p>If you are only using <a href="/docs/26.0.0/ingestion/hadoop">Hadoop-based batch ingestion</a> with no other ingestion types, you can lower the amount of resources allocated per Task. Batch ingestion tasks do not need to answer queries, and the bulk of the ingestion workload will be executed on the Hadoop cluster, so the Tasks do not require much resources.</p><h6 class="anchor anchorWithStickyNavbar_LWe7" id="parallel-native-ingestion">Parallel native ingestion<a href="#parallel-native-ingestion" class="hash-link" aria-label="Direct link to Parallel native ingestion" title="Direct link to Parallel native ingestion"></a></h6><p>If you are using <a href="/docs/26.0.0/ingestion/native-batch">parallel native batch ingestion</a>, allocating more available task slots is a good idea and will allow greater ingestion concurrency.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="coordinator">Coordinator<a href="#coordinator" class="hash-link" aria-label="Direct link to Coordinator" title="Direct link to Coordinator"></a></h3><p>The main performance-related setting on the Coordinator is the heap size.</p><p>The heap requirements of the Coordinator scale with the number of servers, segments, and tasks in the cluster.</p><p>You can set the Coordinator heap to the same size as your Broker heap, or slightly smaller: both services have to process cluster-wide state and answer API requests about this state.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="dynamic-configuration">Dynamic Configuration<a href="#dynamic-configuration" class="hash-link" aria-label="Direct link to Dynamic Configuration" title="Direct link to Dynamic Configuration"></a></h4><p><code>percentOfSegmentsToConsiderPerMove</code></p><ul><li>The default value is 100. This means that the Coordinator will consider all segments when it is looking for a segment to move. The Coordinator makes a weighted choice, with segments on Servers with the least capacity being the most likely segments to be moved.<ul><li>This weighted selection strategy means that the segments on the servers who have the most available capacity are the least likely to be chosen.</li><li>As the number of segments in the cluster increases, the probability of choosing the Nth segment to move decreases; where N is the last segment considered for moving.</li><li>An admin can use this config to skip consideration of that Nth segment.</li></ul></li><li>Instead of skipping a precise amount of segments, we skip a percentage of segments in the cluster.<ul><li>For example, with the value set to 25, only the first 25% of segments will be considered as a segment that can be moved. This 25% of segments will come from the servers that have the least available capacity.<ul><li>In this example, each time the Coordinator looks for a segment to move, it will consider 75% less segments than it did when the configuration was 100. On clusters with hundreds of thousands of segments, this can add up to meaningful coordination time savings.</li></ul></li></ul></li><li>General recommendations for this configuration:<ul><li>If you are not worried about the amount of time it takes your Coordinator to complete a full coordination cycle, you likely do not need to modify this config.</li><li>If you are frustrated with how long the Coordinator takes to run a full coordination cycle, and you have set the Coordinator dynamic config <code>maxSegmentsToMove</code> to a value above 0 (the default is 5), setting this config to a non-default value can help shorten coordination time.<ul><li>The recommended starting point value is 66. It represents a meaningful decrease in the percentage of segments considered while also not being too aggressive (You will consider 1/3 fewer segments per move operation with this value).</li></ul></li></ul></li><li>The impact that modifying this config will have on your coordination time will be a function of how low you set the config value, the value for <code>maxSegmentsToMove</code> and the total number of segments in your cluster.<ul><li>If your cluster has a relatively small number of segments, or you choose to move few segments per coordination cycle, there may not be much savings to be had here.</li></ul></li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="overlord">Overlord<a href="#overlord" class="hash-link" aria-label="Direct link to Overlord" title="Direct link to Overlord"></a></h3><p>The main performance-related setting on the Overlord is the heap size.</p><p>The heap requirements of the Overlord scale primarily with the number of running Tasks.</p><p>The Overlord tends to require less resources than the Coordinator or Broker. You can generally set the Overlord heap to a value that's 25-50% of your Coordinator heap.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="router">Router<a href="#router" class="hash-link" aria-label="Direct link to Router" title="Direct link to Router"></a></h3><p>The Router has light resource requirements, as it proxies requests to Brokers without performing much computational work itself.</p><p>You can assign it 256MiB heap as a starting point, growing it if needed.</p><a name="processing-threads-buffers"></a><h2 class="anchor anchorWithStickyNavbar_LWe7" id="guidelines-for-processing-threads-and-buffers">Guidelines for processing threads and buffers<a href="#guidelines-for-processing-threads-and-buffers" class="hash-link" aria-label="Direct link to Guidelines for processing threads and buffers" title="Direct link to Guidelines for processing threads and buffers"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="processing-threads">Processing threads<a href="#processing-threads" class="hash-link" aria-label="Direct link to Processing threads" title="Direct link to Processing threads"></a></h3><p>The <code>druid.processing.numThreads</code> configuration controls the size of the processing thread pool used for computing query results. The size of this pool limits how many queries can be concurrently processed.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="processing-buffers">Processing buffers<a href="#processing-buffers" class="hash-link" aria-label="Direct link to Processing buffers" title="Direct link to Processing buffers"></a></h3><p><code>druid.processing.buffer.sizeBytes</code> is a closely related property that controls the size of the off-heap buffers allocated to the processing threads.</p><p>One buffer is allocated for each processing thread. A size between 500MiB and 1GiB is a reasonable choice for general use.</p><p>The TopN and GroupBy queries use these buffers to store intermediate computed results. As the buffer size increases, more data can be processed in a single pass.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="groupby-merging-buffers">GroupBy merging buffers<a href="#groupby-merging-buffers" class="hash-link" aria-label="Direct link to GroupBy merging buffers" title="Direct link to GroupBy merging buffers"></a></h3><p>If you plan to issue GroupBy V2 queries, <code>druid.processing.numMergeBuffers</code> is an important configuration property.</p><p>GroupBy V2 queries use an additional pool of off-heap buffers for merging query results. These buffers have the same size as the processing buffers described above, set by the <code>druid.processing.buffer.sizeBytes</code> property.</p><p>Non-nested GroupBy V2 queries require 1 merge buffer per query, while a nested GroupBy V2 query requires 2 merge buffers (regardless of the depth of nesting).</p><p>The number of merge buffers determines the number of GroupBy V2 queries that can be processed concurrently.</p><a name="connection-pool"></a><h2 class="anchor anchorWithStickyNavbar_LWe7" id="connection-pool-guidelines">Connection pool guidelines<a href="#connection-pool-guidelines" class="hash-link" aria-label="Direct link to Connection pool guidelines" title="Direct link to Connection pool guidelines"></a></h2><p>Each Druid process has a configuration property for the number of HTTP connection handling threads, <code>druid.server.http.numThreads</code>.</p><p>The number of HTTP server threads limits how many concurrent HTTP API requests a given process can handle.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sizing-the-connection-pool-for-queries">Sizing the connection pool for queries<a href="#sizing-the-connection-pool-for-queries" class="hash-link" aria-label="Direct link to Sizing the connection pool for queries" title="Direct link to Sizing the connection pool for queries"></a></h3><p>The Broker has a setting <code>druid.broker.http.numConnections</code> that controls how many outgoing connections it can make to a given Historical or Task process.</p><p>These connections are used to send queries to the Historicals or Tasks, with one connection per query; the value of <code>druid.broker.http.numConnections</code> is effectively a limit on the number of concurrent queries that a given broker can process.</p><p>Suppose we have a cluster with 3 Brokers and <code>druid.broker.http.numConnections</code> is set to 10.</p><p>This means that each Broker in the cluster will open up to 10 connections to each individual Historical or Task (for a total of 30 incoming query connections per Historical/Task).</p><p>On the Historical/Task side, this means that <code>druid.server.http.numThreads</code> must be set to a value at least as high as the sum of <code>druid.broker.http.numConnections</code> across all the Brokers in the cluster.</p><p>In practice, you will want to allocate additional server threads for non-query API requests such as status checks; adding 10 threads for those is a good general guideline. Using the example with 3 Brokers in the cluster and <code>druid.broker.http.numConnections</code> set to 10, a value of 40 would be appropriate for <code>druid.server.http.numThreads</code> on Historicals and Tasks.</p><p>As a starting point, allowing for 50 concurrent queries (requests that read segment data from datasources) + 10 non-query requests (other requests like status checks) on Historicals and Tasks is reasonable (i.e., set <code>druid.server.http.numThreads</code> to 60 there), while sizing <code>druid.broker.http.numConnections</code> based on the number of Brokers in the cluster to fit within the 50 query connection limit per Historical/Task.</p><ul><li>If the connection pool across Brokers and Historicals/Tasks is too small, the cluster will be underutilized as there are too few concurrent query slots.</li><li>If the connection pool is too large, you may get out-of-memory errors due to excessive concurrent load, and increased resource contention.</li><li>The connection pool sizing matters most when you require QoS-type guarantees and use query priorities; otherwise, these settings can be more loosely configured.</li><li>If your cluster usage patterns are heavily biased towards a high number of small concurrent queries (where each query takes less than ~15ms), enlarging the connection pool can be a good idea.</li><li>The 50/10 general guideline here is a rough starting point, since different queries impose different amounts of load on the system. To size the connection pool more exactly for your cluster, you would need to know the execution times for your queries and ensure that the rate of incoming queries does not exceed your "drain" rate.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="per-segment-direct-memory-buffers">Per-segment direct memory buffers<a href="#per-segment-direct-memory-buffers" class="hash-link" aria-label="Direct link to Per-segment direct memory buffers" title="Direct link to Per-segment direct memory buffers"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="segment-decompression">Segment decompression<a href="#segment-decompression" class="hash-link" aria-label="Direct link to Segment decompression" title="Direct link to Segment decompression"></a></h3><p>When opening a segment for reading during segment merging or query processing, Druid allocates a 64KiB off-heap decompression buffer for each column being read.</p><p>Thus, there is additional direct memory overhead of (64KiB <em> number of columns read per segment </em> number of segments read) when reading segments.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="segment-merging">Segment merging<a href="#segment-merging" class="hash-link" aria-label="Direct link to Segment merging" title="Direct link to Segment merging"></a></h3><p>In addition to the segment decompression overhead described above, when a set of segments are merged during ingestion, a direct buffer is allocated for every String typed column, for every segment in the set to be merged.</p><p>The size of these buffers are equal to the cardinality of the String column within its segment, times 4 bytes (the buffers store integers).</p><p>For example, if two segments are being merged, the first segment having a single String column with cardinality 1000, and the second segment having a String column with cardinality 500, the merge step would allocate (1000 + 500) * 4 = 6000 bytes of direct memory.</p><p>These buffers are used for merging the value dictionaries of the String column across segments. These "dictionary merging buffers" are independent of the "merge buffers" configured by <code>druid.processing.numMergeBuffers</code>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="general-recommendations">General recommendations<a href="#general-recommendations" class="hash-link" aria-label="Direct link to General recommendations" title="Direct link to General recommendations"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="jvm-tuning">JVM tuning<a href="#jvm-tuning" class="hash-link" aria-label="Direct link to JVM tuning" title="Direct link to JVM tuning"></a></h3><h4 class="anchor anchorWithStickyNavbar_LWe7" id="garbage-collection">Garbage Collection<a href="#garbage-collection" class="hash-link" aria-label="Direct link to Garbage Collection" title="Direct link to Garbage Collection"></a></h4><p>We recommend using the G1GC garbage collector:</p><p><code>-XX:+UseG1GC</code></p><p>Enabling process termination on out-of-memory errors is useful as well, since the process generally will not recover from such a state, and it's better to restart the process:</p><p><code>-XX:+ExitOnOutOfMemoryError</code></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="other-generally-useful-jvm-flags">Other generally useful JVM flags<a href="#other-generally-useful-jvm-flags" class="hash-link" aria-label="Direct link to Other generally useful JVM flags" title="Direct link to Other generally useful JVM flags"></a></h4><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">-Duser.timezone=UTC</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">-Dfile.encoding=UTF-8</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">-Djava.io.tmpdir=<should not be volatile tmpfs and also has good read and write speed. Strongly recommended to avoid using NFS mount></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">-Dorg.jboss.logging.provider=slf4j</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">-Dnet.spy.log.LoggerImpl=net.spy.memcached.compat.log.SLF4JLogger</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">-Dlog4j.shutdownCallbackRegistry=org.apache.druid.common.config.Log4jShutdown</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">-Dlog4j.shutdownHookEnabled=true</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">-XX:+PrintGCDetails</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">-XX:+PrintGCDateStamps</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">-XX:+PrintGCTimeStamps</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">-XX:+PrintGCApplicationStoppedTime</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">-XX:+PrintGCApplicationConcurrentTime</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">-Xloggc:/var/logs/druid/historical.gc.log</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">-XX:+UseGCLogFileRotation</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">-XX:NumberOfGCLogFiles=50</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">-XX:GCLogFileSize=10m</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">-XX:+ExitOnOutOfMemoryError</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">-XX:+HeapDumpOnOutOfMemoryError</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">-XX:HeapDumpPath=/var/logs/druid/historical.hprof</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">-XX:MaxDirectMemorySize=1g</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><blockquote><p>Please note that the flag settings above represent sample, general guidelines only. Be careful to use values appropriate |
| for your specific scenario and be sure to test any changes in staging environments.</p></blockquote><p><code>ExitOnOutOfMemoryError</code> flag is only supported starting JDK 8u92 . For older versions, <code>-XX:OnOutOfMemoryError='kill -9 %p'</code> can be used.</p><p><code>MaxDirectMemorySize</code> restricts JVM from allocating more than specified limit, by setting it to unlimited JVM restriction is lifted and OS level memory limits would still be effective. It's still important to make sure that Druid is not configured to allocate more off-heap memory than your machine has available. Important settings here include <code>druid.processing.numThreads</code>, <code>druid.processing.numMergeBuffers</code>, and <code>druid.processing.buffer.sizeBytes</code>.</p><p>Additionally, for large JVM heaps, here are a few Garbage Collection efficiency guidelines that have been known to help in some cases.</p><ul><li>Mount /tmp on tmpfs. See <a href="http://www.evanjones.ca/jvm-mmap-pause.html" target="_blank" rel="noopener noreferrer">The Four Month Bug: JVM statistics cause garbage collection pauses</a>.</li><li>On Disk-IO intensive processes (e.g., Historical and MiddleManager), GC and Druid logs should be written to a different disk than where data is written.</li><li>Disable <a href="https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html" target="_blank" rel="noopener noreferrer">Transparent Huge Pages</a>.</li><li>Try disabling biased locking by using <code>-XX:-UseBiasedLocking</code> JVM flag. See <a href="https://dzone.com/articles/logging-stop-world-pauses-jvm" target="_blank" rel="noopener noreferrer">Logging Stop-the-world Pauses in JVM</a>.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="use-utc-timezone">Use UTC timezone<a href="#use-utc-timezone" class="hash-link" aria-label="Direct link to Use UTC timezone" title="Direct link to Use UTC timezone"></a></h3><p>We recommend using UTC timezone for all your events and across your hosts, not just for Druid, but for all data infrastructure. This can greatly mitigate potential query problems with inconsistent timezones. To query in a non-UTC timezone see <a href="/docs/26.0.0/querying/granularities#period-granularities">query granularities</a></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="system-configuration">System configuration<a href="#system-configuration" class="hash-link" aria-label="Direct link to System configuration" title="Direct link to System configuration"></a></h3><h4 class="anchor anchorWithStickyNavbar_LWe7" id="ssds">SSDs<a href="#ssds" class="hash-link" aria-label="Direct link to SSDs" title="Direct link to SSDs"></a></h4><p>SSDs are highly recommended for Historical, MiddleManager, and Indexer processes if you are not running a cluster that is entirely in memory. SSDs can greatly mitigate the time required to page data in and out of memory.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="jbod-vs-raid">JBOD vs RAID<a href="#jbod-vs-raid" class="hash-link" aria-label="Direct link to JBOD vs RAID" title="Direct link to JBOD vs RAID"></a></h4><p>Historical processes store large number of segments on Disk and support specifying multiple paths for storing those. Typically, hosts have multiple disks configured with RAID which makes them look like a single disk to OS. RAID might have overheads specially if its not hardware controller based but software based. So, Historicals might get improved disk throughput with JBOD.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="swap-space">Swap space<a href="#swap-space" class="hash-link" aria-label="Direct link to Swap space" title="Direct link to Swap space"></a></h4><p>We recommend <em>not</em> using swap space for Historical, MiddleManager, and Indexer processes since due to the large number of memory mapped segment files can lead to poor and unpredictable performance.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="linux-limits">Linux limits<a href="#linux-limits" class="hash-link" aria-label="Direct link to Linux limits" title="Direct link to Linux limits"></a></h4><p>For Historical, MiddleManager, and Indexer processes (and for really large clusters, Broker processes), you might need to adjust some Linux system limits to account for a large number of open files, a large number of network connections, or a large number of memory mapped files.</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="ulimit">ulimit<a href="#ulimit" class="hash-link" aria-label="Direct link to ulimit" title="Direct link to ulimit"></a></h5><p>The limit on the number of open files can be set permanently by editing <code>/etc/security/limits.conf</code>. This value should be substantially greater than the number of segment files that will exist on the server.</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="max_map_count">max_map_count<a href="#max_map_count" class="hash-link" aria-label="Direct link to max_map_count" title="Direct link to max_map_count"></a></h5><p>Historical processes and to a lesser extent, MiddleManager and Indexer processes memory map segment files, so depending on the number of segments per server, <code>/proc/sys/vm/max_map_count</code> might also need to be adjusted. Depending on the variant of Linux, this might be done via <code>sysctl</code> by placing a file in <code>/etc/sysctl.d/</code> that sets <code>vm.max_map_count</code>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/docs/26.0.0/operations/tls-support"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">TLS support</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/docs/26.0.0/operations/segment-optimization"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Segment size optimization</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#process-specific-guidelines" class="table-of-contents__link toc-highlight">Process-specific guidelines</a><ul><li><a href="#historical" class="table-of-contents__link toc-highlight">Historical</a></li><li><a href="#broker" class="table-of-contents__link toc-highlight">Broker</a></li><li><a href="#middlemanager" class="table-of-contents__link toc-highlight">MiddleManager</a></li><li><a href="#coordinator" class="table-of-contents__link toc-highlight">Coordinator</a></li><li><a href="#overlord" class="table-of-contents__link toc-highlight">Overlord</a></li><li><a href="#router" class="table-of-contents__link toc-highlight">Router</a></li></ul></li><li><a href="#guidelines-for-processing-threads-and-buffers" class="table-of-contents__link toc-highlight">Guidelines for processing threads and buffers</a><ul><li><a href="#processing-threads" class="table-of-contents__link toc-highlight">Processing threads</a></li><li><a href="#processing-buffers" class="table-of-contents__link toc-highlight">Processing buffers</a></li><li><a href="#groupby-merging-buffers" class="table-of-contents__link toc-highlight">GroupBy merging buffers</a></li></ul></li><li><a href="#connection-pool-guidelines" class="table-of-contents__link toc-highlight">Connection pool guidelines</a><ul><li><a href="#sizing-the-connection-pool-for-queries" class="table-of-contents__link toc-highlight">Sizing the connection pool for queries</a></li></ul></li><li><a href="#per-segment-direct-memory-buffers" class="table-of-contents__link toc-highlight">Per-segment direct memory buffers</a><ul><li><a href="#segment-decompression" class="table-of-contents__link toc-highlight">Segment decompression</a></li><li><a href="#segment-merging" class="table-of-contents__link toc-highlight">Segment merging</a></li></ul></li><li><a href="#general-recommendations" class="table-of-contents__link toc-highlight">General recommendations</a><ul><li><a href="#jvm-tuning" class="table-of-contents__link toc-highlight">JVM tuning</a></li><li><a href="#use-utc-timezone" class="table-of-contents__link toc-highlight">Use UTC timezone</a></li><li><a href="#system-configuration" class="table-of-contents__link toc-highlight">System configuration</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer"><div class="container container-fluid"><div class="footer__bottom text--center"><div class="margin-bottom--sm"><img src="/img/favicon.png" class="themedImage_ToTc themedImage--light_HNdA footer__logo"><img src="/img/favicon.png" class="themedImage_ToTc themedImage--dark_i4oU footer__logo"></div><div class="footer__copyright">Copyright © 2023 Apache Software Foundation. Except where otherwise noted, licensed under CC BY-SA 4.0. Apache Druid, Druid, and the Druid logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</div></div></div></footer></div> |
| <script src="/assets/js/runtime~main.38900cbf.js"></script> |
| <script src="/assets/js/main.5e106d68.js"></script> |
| </body> |
| </html> |
