| <!doctype html> |
| <html lang="en" dir="ltr" class="docs-wrapper plugin-docs plugin-id-default docs-version-current docs-doc-page docs-doc-id-release-info/upgrade-notes" data-has-hydrated="false"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="generator" content="Docusaurus v3.7.0"> |
| <title data-rh="true">Upgrade notes | Apache® Druid</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:image" content="https://druid.apache.org/img/druid_nav.png"><meta data-rh="true" name="twitter:image" content="https://druid.apache.org/img/druid_nav.png"><meta data-rh="true" property="og:url" content="https://druid.apache.org/docs/latest/release-info/upgrade-notes"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="current"><meta data-rh="true" name="docusaurus_tag" content="docs-default-current"><meta data-rh="true" name="docsearch:version" content="current"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-current"><meta data-rh="true" property="og:title" content="Upgrade notes | Apache® Druid"><meta data-rh="true" name="description" content="<!--"><meta data-rh="true" property="og:description" content="<!--"><link data-rh="true" rel="icon" href="/img/favicon.png"><link data-rh="true" rel="canonical" href="https://druid.apache.org/docs/latest/release-info/upgrade-notes"><link data-rh="true" rel="alternate" href="https://druid.apache.org/docs/latest/release-info/upgrade-notes" hreflang="en"><link data-rh="true" rel="alternate" href="https://druid.apache.org/docs/latest/release-info/upgrade-notes" hreflang="x-default"><link rel="stylesheet" href="/css/all.css"> |
| <script src="/js/clipboard.min.js"></script><link rel="stylesheet" href="/assets/css/styles.f3140859.css"> |
| <script src="/assets/js/runtime~main.5b007a36.js" defer="defer"></script> |
| <script src="/assets/js/main.7b126984.js" defer="defer"></script> |
| </head> |
| <body class="navigation-with-keyboard"> |
| <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return window.localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const n=new URLSearchParams(window.location.search).entries();for(var[t,e]of n)if(t.startsWith("docusaurus-data-")){var a=t.replace("docusaurus-data-","data-");document.documentElement.setAttribute(a,e)}}catch(t){}}()</script><div id="__docusaurus"><div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><nav aria-label="Main" class="navbar navbar--fixed-top navbar--dark"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/druid_nav.png" alt="Apache® Druid" class="themedComponent_mlkZ themedComponent--light_NVdE"><img src="/img/druid_nav.png" alt="Apache® Druid" class="themedComponent_mlkZ themedComponent--dark_xIcU"></div></a></div><div class="navbar__items navbar__items--right"><a class="navbar__item navbar__link" href="/technology">Technology</a><a class="navbar__item navbar__link" href="/use-cases">Use Cases</a><a class="navbar__item navbar__link" href="/druid-powered">Powered By</a><a class="navbar__item navbar__link" href="/docs/latest/design/">Docs</a><a class="navbar__item navbar__link" href="/community/">Community</a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Apache®</a><ul class="dropdown__menu"><li><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Foundation<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://apachecon.com/?ref=druid.apache.org" target="_blank" rel="noopener noreferrer" class="dropdown__link">Events<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/licenses/" target="_blank" rel="noopener noreferrer" class="dropdown__link">License<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Thanks<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/security/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Security<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Sponsorship<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a class="navbar__item navbar__link" href="/downloads/">Download</a><div class="navbarSearchContainer_Bca1"><div class="navbar__search"><span aria-label="expand searchbar" role="button" class="search-icon" tabindex="0"></span><input id="search_input_react" type="search" placeholder="Loading..." aria-label="Search" class="navbar__search-input search-bar" disabled=""></div></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0"><div class="docsWrapper_hBAB"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docRoot_UBD9"><aside class="theme-doc-sidebar-container docSidebarContainer_YfHR"><div class="sidebarViewport_aRkj"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/latest/design/">Introduction to Apache Druid</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" href="/docs/latest/tutorials/">Getting started</a><button aria-label="Expand sidebar category 'Getting started'" aria-expanded="false" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" role="button" aria-expanded="false" href="/docs/latest/design/architecture">Design</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" href="/docs/latest/ingestion/">Ingestion</a><button aria-label="Expand sidebar category 'Ingestion'" aria-expanded="false" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" href="/docs/latest/data-management/">Data management</a><button aria-label="Expand sidebar category 'Data management'" aria-expanded="false" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" role="button" aria-expanded="false" href="/docs/latest/querying/sql">Querying</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" href="/docs/latest/api-reference/">API reference</a><button aria-label="Expand sidebar category 'API reference'" aria-expanded="false" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" href="/docs/latest/configuration/">Configuration</a><button aria-label="Expand sidebar category 'Configuration'" aria-expanded="false" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" role="button" aria-expanded="false" href="/docs/latest/api-reference/automatic-compaction-api">Operations</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" href="/docs/latest/development/overview">Development</a><button aria-label="Expand sidebar category 'Development'" aria-expanded="false" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" role="button" aria-expanded="true" href="/docs/latest/release-info/release-notes">Release info</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/latest/release-info/release-notes">Release notes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/docs/latest/release-info/upgrade-notes">Upgrade notes</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" tabindex="0" href="/docs/latest/release-info/migration-guide">Migration guides</a><button aria-label="Expand sidebar category 'Migration guides'" aria-expanded="false" type="button" class="clean-btn menu__caret"></button></div></li></ul></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/latest/misc/papers-and-talks">Papers</a></li></ul></nav></div></div></aside><main class="docMainContainer_TBSr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Release info</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Upgrade notes</span><meta itemprop="position" content="2"></li></ul></nav><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Upgrade notes</h1></header><p>The upgrade notes assume that you are upgrading from the Druid version that immediately precedes your target version. If you are upgrading across multiple versions, make sure you read the upgrade notes for all the intermediate versions.</p> |
| <p>For the full release notes for a specific version, see the <a href="https://github.com/apache/druid/releases" target="_blank" rel="noopener noreferrer">releases page</a>.</p> |
| <h2 class="anchor anchorWithStickyNavbar_LWe7" id="3300">33.0.0<a href="#3300" class="hash-link" aria-label="Direct link to 33.0.0" title="Direct link to 33.0.0"></a></h2> |
| <h3 class="anchor anchorWithStickyNavbar_LWe7" id="upgrade-notes">Upgrade notes<a href="#upgrade-notes" class="hash-link" aria-label="Direct link to Upgrade notes" title="Direct link to Upgrade notes"></a></h3> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="usemaxmemoryestimates"><code>useMaxMemoryEstimates</code><a href="#usemaxmemoryestimates" class="hash-link" aria-label="Direct link to usemaxmemoryestimates" title="Direct link to usemaxmemoryestimates"></a></h4> |
| <p><code>useMaxMemoryEstimates</code> is now set to false for MSQ task engine tasks. Additionally, the property has been deprecated and will be removed in a future release. Setting this to false allows for better on-heap memory estimation.</p> |
| <p><a href="https://github.com/apache/druid/pull/17792" target="_blank" rel="noopener noreferrer">#17792</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="automatic-kill-tasks-interval">Automatic kill tasks interval<a href="#automatic-kill-tasks-interval" class="hash-link" aria-label="Direct link to Automatic kill tasks interval" title="Direct link to Automatic kill tasks interval"></a></h4> |
| <p>Automatic kill tasks are now limited to 30 days or fewer worth of segments per task.</p> |
| <p>The previous behavior (no limit on interval per kill task) can be restored by setting <code>druid.coordinator.kill.maxInterval = P0D</code>.</p> |
| <p><a href="https://github.com/apache/druid/pull/17680" target="_blank" rel="noopener noreferrer">#17680</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="kubernetes-deployments">Kubernetes deployments<a href="#kubernetes-deployments" class="hash-link" aria-label="Direct link to Kubernetes deployments" title="Direct link to Kubernetes deployments"></a></h4> |
| <p>By default, the Docker image now uses the canonical hostname if you're running Druid in Kubernetes. Otherwise, it uses the IP address otherwise <a href="https://github.com/apache/druid/pull/17697" target="_blank" rel="noopener noreferrer">#17697</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="updated-configs">Updated configs<a href="#updated-configs" class="hash-link" aria-label="Direct link to Updated configs" title="Direct link to Updated configs"></a></h4> |
| <p>Various configs were deprecated in a previous release and have now been removed. The following table lists the removed configs and their replacements:</p> |
| <table><thead><tr><th>Removed config</th><th>Replacement config</th></tr></thead><tbody><tr><td><code>druid.processing.merge.task.initialYieldNumRows </code></td><td><code>druid.processing.merge.initialYieldNumRows</code></td></tr><tr><td><code>druid.processing.merge.task.targetRunTimeMillis</code></td><td><code>druid.processing.merge.targetRunTimeMillis</code></td></tr><tr><td><code>druid.processing.merge.task.smallBatchNumRows</code></td><td><code>druid.processing.merge.smallBatchNumRows</code></td></tr><tr><td><code>druid.processing.merge.pool.awaitShutdownMillis</code></td><td></td></tr><tr><td><code>druid.processing.merge.awaitShutdownMillis</code></td><td></td></tr><tr><td><code>druid.processing.merge.pool.parallelism</code></td><td><code>druid.processing.merge.parallelism</code></td></tr><tr><td><code>druid.processing.merge.pool.defaultMaxQueryParallelism</code></td><td><code>druid.processing.merge.defaultMaxQueryParallelism</code></td></tr></tbody></table> |
| <p><a href="https://github.com/apache/druid/pull/17776" target="_blank" rel="noopener noreferrer">#17776</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="segment-metadata-cache-configs">Segment metadata cache configs<a href="#segment-metadata-cache-configs" class="hash-link" aria-label="Direct link to Segment metadata cache configs" title="Direct link to Segment metadata cache configs"></a></h4> |
| <p>If you need to downgrade to a version where Druid doesn't support the segment metadata cache, you must set the <code>druid.manager.segments.useCache</code> config to false or remove it prior to the upgrade.</p> |
| <p>This feature is introduced in Druid 33.0.</p> |
| <p><a href="https://github.com/apache/druid/pull/17653" target="_blank" rel="noopener noreferrer">#17653</a></p> |
| <h2 class="anchor anchorWithStickyNavbar_LWe7" id="3200">32.0.0<a href="#3200" class="hash-link" aria-label="Direct link to 32.0.0" title="Direct link to 32.0.0"></a></h2> |
| <h3 class="anchor anchorWithStickyNavbar_LWe7" id="incompatible-changes">Incompatible changes<a href="#incompatible-changes" class="hash-link" aria-label="Direct link to Incompatible changes" title="Direct link to Incompatible changes"></a></h3> |
| <h3 class="anchor anchorWithStickyNavbar_LWe7" id="ansi-sql-compatibility-and-query-results">ANSI-SQL compatibility and query results<a href="#ansi-sql-compatibility-and-query-results" class="hash-link" aria-label="Direct link to ANSI-SQL compatibility and query results" title="Direct link to ANSI-SQL compatibility and query results"></a></h3> |
| <p>Support for the configs that let you maintain older behavior that wasn't ANSI-SQL compliant have been removed:</p> |
| <ul> |
| <li><code>druid.generic.useDefaultValueForNull=true</code></li> |
| <li><code>druid.expressions.useStrictBooleans=false</code></li> |
| <li><code>druid.generic.useThreeValueLogicForNativeFilters=false</code></li> |
| </ul> |
| <p>They no longer affect your query results. Only SQL-compliant non-legacy behavior is supported now.</p> |
| <p>If the configs are set to the legacy behavior, Druid services will fail to start.</p> |
| <p>If you want to continue to get the same results without these settings, you must update your queries or your results will be incorrect after you upgrade.</p> |
| <p>For more information about how to update your queries, see the <a href="https://druid.apache.org/docs/latest/release-info/migr-ansi-sql-null" target="_blank" rel="noopener noreferrer">migration guide</a>.</p> |
| <p><a href="https://github.com/apache/druid/pull/17568" target="_blank" rel="noopener noreferrer">#17568</a> <a href="https://github.com/apache/druid/pull/17609" target="_blank" rel="noopener noreferrer">#17609</a></p> |
| <h3 class="anchor anchorWithStickyNavbar_LWe7" id="java-support">Java support<a href="#java-support" class="hash-link" aria-label="Direct link to Java support" title="Direct link to Java support"></a></h3> |
| <p>Java support in Druid has been updated:</p> |
| <ul> |
| <li>Java 8 support has been removed</li> |
| <li>Java 11 support is deprecated</li> |
| </ul> |
| <p>We recommend that you upgrade to Java 17.</p> |
| <p><a href="https://github.com/apache/druid/pull/17466" target="_blank" rel="noopener noreferrer">#17466</a></p> |
| <h3 class="anchor anchorWithStickyNavbar_LWe7" id="javascript-support">Javascript support<a href="#javascript-support" class="hash-link" aria-label="Direct link to Javascript support" title="Direct link to Javascript support"></a></h3> |
| <ul> |
| <li>Javascript tiered broker selector strategy and Javascript filters currently do not work on Java 17.</li> |
| </ul> |
| <h3 class="anchor anchorWithStickyNavbar_LWe7" id="deprecations">Deprecations<a href="#deprecations" class="hash-link" aria-label="Direct link to Deprecations" title="Direct link to Deprecations"></a></h3> |
| <h3 class="anchor anchorWithStickyNavbar_LWe7" id="hadoop-based-ingestion">Hadoop-based ingestion<a href="#hadoop-based-ingestion" class="hash-link" aria-label="Direct link to Hadoop-based ingestion" title="Direct link to Hadoop-based ingestion"></a></h3> |
| <p>Hadoop-based ingestion is now deprecated. We recommend that you migrate to SQL-based ingestion.</p> |
| <h2 class="anchor anchorWithStickyNavbar_LWe7" id="3100">31.0.0<a href="#3100" class="hash-link" aria-label="Direct link to 31.0.0" title="Direct link to 31.0.0"></a></h2> |
| <h3 class="anchor anchorWithStickyNavbar_LWe7" id="upgrade-notes-1">Upgrade notes<a href="#upgrade-notes-1" class="hash-link" aria-label="Direct link to Upgrade notes" title="Direct link to Upgrade notes"></a></h3> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="array-ingest-mode-now-defaults-to-array">Array ingest mode now defaults to array<a href="#array-ingest-mode-now-defaults-to-array" class="hash-link" aria-label="Direct link to Array ingest mode now defaults to array" title="Direct link to Array ingest mode now defaults to array"></a></h4> |
| <p>The SQL-based ingestion query context flag <code>arrayIngestMode</code> now defaults to <code>array</code> instead of <code>mvd</code>. This means that SQL <code>VARCHAR ARRAY</code> types is no longer implicitly translated and stored in <code>VARCHAR</code> columns, but is instead stored as <code>VARCHAR ARRAY</code>. This change permits other array types such as <code>BIGINT ARRAY</code> and <code>DOUBLE ARRAY</code> to be inserted with MSQ task engine into their respective array column types instead of failing as they do in <code>mvd</code> mode.</p> |
| <p>To continue to store multi-value strings, modify any insert/replace queries to wrap the array types with the <code>ARRAY_TO_MV</code> operator.</p> |
| <p>Validation is in place to prevent mixing <code>VARCHAR</code> and <code>VARCHAR ARRAY</code> columns in the same table, so any ingestions affected by this change will fail and provide a descriptive error message instead of exhibiting unexpected behavior.</p> |
| <p>The <code>arrayIngestMode</code> option of <code>none</code> has been removed. It was introduced prior to the table validation logic as a means for cluster operators to force query writers to explicitly set <code>array</code> or <code>mvd</code> on their query contexts, but provides little utility in Druid 31.</p> |
| <p>See the following topics for more information:</p> |
| <ul> |
| <li><a href="https://druid.apache.org/docs/latest/querying/multi-value-dimensions.md#sql-based-ingestion" target="_blank" rel="noopener noreferrer">Ingest multi-value dimensions</a> for how to ingest multi-value strings.</li> |
| <li><a href="https://druid.apache.org/docs/latest/querying/arrays.md#sql-based-ingestion" target="_blank" rel="noopener noreferrer">Ingest arrays</a> for ingesting arrays.</li> |
| </ul> |
| <p><a href="https://github.com/apache/druid/pull/16789" target="_blank" rel="noopener noreferrer">#16789</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="removed-task-action-audit-logging">Removed task action audit logging<a href="#removed-task-action-audit-logging" class="hash-link" aria-label="Direct link to Removed task action audit logging" title="Direct link to Removed task action audit logging"></a></h4> |
| <p>The deprecated task action audit logging has been removed. This change includes the following updates:</p> |
| <ul> |
| <li>The endpoint <code>/indexer/v1/task/{taskId}/segments</code> is no longer supported.</li> |
| <li>Druid doesn't write to or read from the metadata table <code>druid_taskLog</code>.</li> |
| <li>Druid ignores the property <code>druid.indexer.auditlog.enabled</code>.</li> |
| <li>Druid doesn't emit the metric <code>task/action/log/time</code>.</li> |
| </ul> |
| <p>These changes are backward compatible with all existing metadata storage extensions.</p> |
| <p><a href="https://github.com/apache/druid/pull/16309" target="_blank" rel="noopener noreferrer">#16309</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="removed-firehose-and-firehosefactory">Removed Firehose and FirehoseFactory<a href="#removed-firehose-and-firehosefactory" class="hash-link" aria-label="Direct link to Removed Firehose and FirehoseFactory" title="Direct link to Removed Firehose and FirehoseFactory"></a></h4> |
| <p>Removed Firehose and FirehoseFactory and remaining implementations. |
| Apache deprecated support for Druid firehoses in version 0.17. Support for firehose ingestion was removed in version 26.0.</p> |
| <p><a href="https://github.com/apache/druid/pull/16758" target="_blank" rel="noopener noreferrer">#16758</a></p> |
| <h3 class="anchor anchorWithStickyNavbar_LWe7" id="incompatible-changes-1">Incompatible changes<a href="#incompatible-changes-1" class="hash-link" aria-label="Direct link to Incompatible changes" title="Direct link to Incompatible changes"></a></h3> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="removed-the-scan-query-legacy-mode">Removed the scan query legacy mode<a href="#removed-the-scan-query-legacy-mode" class="hash-link" aria-label="Direct link to Removed the scan query legacy mode" title="Direct link to Removed the scan query legacy mode"></a></h4> |
| <p>The native scan query legacy mode has been removed. It was introduced in Druid 0.11 to maintain compatibility during an upgrade from older versions of Druid where the scan query was part of a <code>contrib</code> extension.</p> |
| <p><a href="https://github.com/apache/druid/pull/16659" target="_blank" rel="noopener noreferrer">#16659</a></p> |
| <p>Hard-coded <code>"legacy":false</code> following removal of the legacy mode to prevent error during rolling upgrades or downgrades.</p> |
| <p><a href="https://github.com/apache/druid/pull/16793" target="_blank" rel="noopener noreferrer">#16793</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="zk-based-segment-loading">ZK-based segment loading<a href="#zk-based-segment-loading" class="hash-link" aria-label="Direct link to ZK-based segment loading" title="Direct link to ZK-based segment loading"></a></h4> |
| <p>ZK-based segment loading is now disabled. ZK <code>servedSegmentsPath</code> was deprecated in Druid 0.7.1. This legacy path has been replaced by <code>liveSegmentsPath</code>.</p> |
| <p>Segment-serving processes such as Peons, Historicals and Indexers no longer create ZK <code>loadQueuePath</code> entries. The <code>druid.zk.paths.loadQueuePath</code> and <code>druid.zk.paths.servedSegmentsPath</code> properties are no longer used.</p> |
| <p>Move to HTTP-based segment loading first and then perform the version upgrade.</p> |
| <h2 class="anchor anchorWithStickyNavbar_LWe7" id="3000">30.0.0<a href="#3000" class="hash-link" aria-label="Direct link to 30.0.0" title="Direct link to 30.0.0"></a></h2> |
| <h3 class="anchor anchorWithStickyNavbar_LWe7" id="upgrade-notes-2">Upgrade notes<a href="#upgrade-notes-2" class="hash-link" aria-label="Direct link to Upgrade notes" title="Direct link to Upgrade notes"></a></h3> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="append-jsonpath-function">Append JsonPath function<a href="#append-jsonpath-function" class="hash-link" aria-label="Direct link to Append JsonPath function" title="Direct link to Append JsonPath function"></a></h4> |
| <p>The <code>append</code> function for JsonPath for ORC format now fails with an exception. Previously, it would run but not append anything.</p> |
| <p><a href="https://github.com/apache/druid/pull/15772" target="_blank" rel="noopener noreferrer">#15772</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="kinesis-ingestion-tuning">Kinesis ingestion tuning<a href="#kinesis-ingestion-tuning" class="hash-link" aria-label="Direct link to Kinesis ingestion tuning" title="Direct link to Kinesis ingestion tuning"></a></h4> |
| <p>The following properties have been deprecated as part of simplifying the memory tuning for Kinesis ingestion:</p> |
| <ul> |
| <li><code>recordBufferSize</code>, use <code>recordBufferSizeBytes</code> instead</li> |
| <li><code>maxRecordsPerPoll</code>, use <code>maxBytesPerPoll</code> instead</li> |
| </ul> |
| <p><a href="https://github.com/apache/druid/pull/15360" target="_blank" rel="noopener noreferrer">#15360</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="improved-supervisor-rolling-restarts">Improved Supervisor rolling restarts<a href="#improved-supervisor-rolling-restarts" class="hash-link" aria-label="Direct link to Improved Supervisor rolling restarts" title="Direct link to Improved Supervisor rolling restarts"></a></h4> |
| <p>The <code>stopTaskCount</code> config now prioritizes stopping older tasks first. As part of this change, you must also explicitly set a value for <code>stopTaskCount</code>. It no longer defaults to the same value as <code>taskCount</code>.</p> |
| <p><a href="https://github.com/apache/druid/pull/15859" target="_blank" rel="noopener noreferrer">#15859</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="changes-to-coordinator-default-values">Changes to Coordinator default values<a href="#changes-to-coordinator-default-values" class="hash-link" aria-label="Direct link to Changes to Coordinator default values" title="Direct link to Changes to Coordinator default values"></a></h4> |
| <p>The following are the changes to the default values for the Coordinator service:</p> |
| <ul> |
| <li>The default value for <code>druid.coordinator.kill.period</code> (if unspecified) has changed from <code>P1D</code> to the value of <code>druid.coordinator.period.indexingPeriod</code>. Operators can choose to override <code>druid.coordinator.kill.period</code> and that takes precedence over the default behavior.</li> |
| <li>The default value for the dynamic configuration property <code>killTaskSlotRatio</code> has been updated from <code>1.0</code> to <code>0.1</code>. This ensures that kill tasks take up only one task slot by default instead of consuming all available task slots.</li> |
| </ul> |
| <p><a href="https://github.com/apache/druid/pull/16247" target="_blank" rel="noopener noreferrer">#16247</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="googletasklogs-upload-buffer-size"><code>GoogleTaskLogs</code> upload buffer size<a href="#googletasklogs-upload-buffer-size" class="hash-link" aria-label="Direct link to googletasklogs-upload-buffer-size" title="Direct link to googletasklogs-upload-buffer-size"></a></h4> |
| <p>Changed the upload buffer size in <code>GoogleTaskLogs</code> to 1 MB instead of 15 MB to allow more uploads in parallel and prevent the Middle Manager service from running out of memory.</p> |
| <p><a href="https://github.com/apache/druid/pull/16236" target="_blank" rel="noopener noreferrer">#16236</a></p> |
| <h3 class="anchor anchorWithStickyNavbar_LWe7" id="incompatible-changes-2">Incompatible changes<a href="#incompatible-changes-2" class="hash-link" aria-label="Direct link to Incompatible changes" title="Direct link to Incompatible changes"></a></h3> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="changes-to-targetdatasource-in-explain-queries">Changes to <code>targetDataSource</code> in EXPLAIN queries<a href="#changes-to-targetdatasource-in-explain-queries" class="hash-link" aria-label="Direct link to changes-to-targetdatasource-in-explain-queries" title="Direct link to changes-to-targetdatasource-in-explain-queries"></a></h4> |
| <p>Druid 30.0.0 includes a breaking change that restores the behavior for <code>targetDataSource</code> to its 28.0.0 and earlier state, different from Druid 29.0.0 and only 29.0.0. In 29.0.0, <code>targetDataSource</code> returns a JSON object that includes the datasource name. In all other versions, <code>targetDataSource</code> returns a string containing the name of the datasource.</p> |
| <p>If you're upgrading from any version other than 29.0.0, there is no change in behavior.</p> |
| <p>If you are upgrading from 29.0.0, this is an incompatible change.</p> |
| <p><a href="https://github.com/apache/druid/pull/16004" target="_blank" rel="noopener noreferrer">#16004</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="removed-zookeeper-based-segment-loading">Removed ZooKeeper-based segment loading<a href="#removed-zookeeper-based-segment-loading" class="hash-link" aria-label="Direct link to Removed ZooKeeper-based segment loading" title="Direct link to Removed ZooKeeper-based segment loading"></a></h4> |
| <p>ZooKeeper-based segment loading is being removed due to known issues. |
| It has been deprecated for several releases. |
| Recent improvements to the Druid Coordinator have significantly enhanced performance with HTTP-based segment loading.</p> |
| <p><a href="https://github.com/apache/druid/pull/15705" target="_blank" rel="noopener noreferrer">#15705</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="removed-coordinator-configs">Removed Coordinator configs<a href="#removed-coordinator-configs" class="hash-link" aria-label="Direct link to Removed Coordinator configs" title="Direct link to Removed Coordinator configs"></a></h4> |
| <p>Removed the following Coordinator configs:</p> |
| <ul> |
| <li><code>druid.coordinator.load.timeout</code>: Not needed as the default value of this parameter (15 minutes) is known to work well for all clusters.</li> |
| <li><code>druid.coordinator.loadqueuepeon.type</code>: Not needed as this value is always <code>http</code>.</li> |
| <li><code>druid.coordinator.curator.loadqueuepeon.numCallbackThreads</code>: Not needed as ZooKeeper(curator)-based segment loading isn't an option anymore.</li> |
| </ul> |
| <p>Auto-cleanup of compaction configs of inactive datasources is now enabled by default.</p> |
| <p><a href="https://github.com/apache/druid/pull/15705" target="_blank" rel="noopener noreferrer">#15705</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="changed-usemaxmemoryestimates-for-hadoop-jobs">Changed <code>useMaxMemoryEstimates</code> for Hadoop jobs<a href="#changed-usemaxmemoryestimates-for-hadoop-jobs" class="hash-link" aria-label="Direct link to changed-usemaxmemoryestimates-for-hadoop-jobs" title="Direct link to changed-usemaxmemoryestimates-for-hadoop-jobs"></a></h4> |
| <p>The default value of the <code>useMaxMemoryEstimates</code> parameter for Hadoop jobs is now <code>false</code>.</p> |
| <p><a href="https://github.com/apache/druid/pull/16280" target="_blank" rel="noopener noreferrer">#16280</a></p> |
| <h2 class="anchor anchorWithStickyNavbar_LWe7" id="2901">29.0.1<a href="#2901" class="hash-link" aria-label="Direct link to 29.0.1" title="Direct link to 29.0.1"></a></h2> |
| <h3 class="anchor anchorWithStickyNavbar_LWe7" id="incompatible-changes-3">Incompatible changes<a href="#incompatible-changes-3" class="hash-link" aria-label="Direct link to Incompatible changes" title="Direct link to Incompatible changes"></a></h3> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="changes-to-targetdatasource-in-explain-queries-1">Changes to <code>targetDataSource</code> in EXPLAIN queries<a href="#changes-to-targetdatasource-in-explain-queries-1" class="hash-link" aria-label="Direct link to changes-to-targetdatasource-in-explain-queries-1" title="Direct link to changes-to-targetdatasource-in-explain-queries-1"></a></h4> |
| <p>Druid 29.0.1 includes a breaking change that restores the behavior for <code>targetDataSource</code> to its 28.0.0 and earlier state, different from Druid 29.0.0 and only 29.0.0. In 29.0.0, <code>targetDataSource</code> returns a JSON object that includes the datasource name. In all other versions, <code>targetDataSource</code> returns a string containing the name of the datasource.</p> |
| <p>If you're upgrading from any version other than 29.0.0, there is no change in behavior.</p> |
| <p>If you are upgrading from 29.0.0, this is an incompatible change.</p> |
| <p><a href="https://github.com/apache/druid/pull/16004" target="_blank" rel="noopener noreferrer">#16004</a></p> |
| <h2 class="anchor anchorWithStickyNavbar_LWe7" id="2900">29.0.0<a href="#2900" class="hash-link" aria-label="Direct link to 29.0.0" title="Direct link to 29.0.0"></a></h2> |
| <h3 class="anchor anchorWithStickyNavbar_LWe7" id="upgrade-notes-3">Upgrade notes<a href="#upgrade-notes-3" class="hash-link" aria-label="Direct link to Upgrade notes" title="Direct link to Upgrade notes"></a></h3> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="changed-equals-filter-for-native-queries">Changed <code>equals</code> filter for native queries<a href="#changed-equals-filter-for-native-queries" class="hash-link" aria-label="Direct link to changed-equals-filter-for-native-queries" title="Direct link to changed-equals-filter-for-native-queries"></a></h4> |
| <p>The <a href="https://druid.apache.org/docs/latest/querying/filters#equality-filter" target="_blank" rel="noopener noreferrer">equality filter</a> on mixed type <code>auto</code> columns that contain arrays must now be filtered as their presenting type. This means that if any rows are arrays (for example, the segment metadata and <code>information_schema</code> reports the type as some array type), then the native queries must also filter as if they are some array type.</p> |
| <p>This change impacts mixed type <code>auto</code> columns that contain both scalars and arrays. It doesn't impact SQL, which already has this limitation due to how the type presents itself.</p> |
| <p><a href="https://github.com/apache/druid/pull/15503" target="_blank" rel="noopener noreferrer">#15503</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="console-automatically-sets-arrayingestmode-for-msq-queries">Console automatically sets <code>arrayIngestMode</code> for MSQ queries<a href="#console-automatically-sets-arrayingestmode-for-msq-queries" class="hash-link" aria-label="Direct link to console-automatically-sets-arrayingestmode-for-msq-queries" title="Direct link to console-automatically-sets-arrayingestmode-for-msq-queries"></a></h4> |
| <p>Druid console now configures the <code>arrayIngestMode</code> parameter in the data loading flow, and its value can persist across the SQL tab unless manually updated. When loading multi-value dimensions or arrays in the Druid console, note the value of the <code>arrayIngestMode</code> parameter to prevent mixing multi-value dimensions and arrays in the same column of a data source.</p> |
| <p><a href="https://github.com/apache/druid/pull/15588" target="_blank" rel="noopener noreferrer">#15588</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="improved-concurrent-append-and-replace-experimental">Improved concurrent append and replace (experimental)<a href="#improved-concurrent-append-and-replace-experimental" class="hash-link" aria-label="Direct link to Improved concurrent append and replace (experimental)" title="Direct link to Improved concurrent append and replace (experimental)"></a></h4> |
| <p>You no longer have to manually determine the task lock type for concurrent append and replace (experimental) with the <code>taskLockType</code> task context. Instead, Druid can now determine it automatically for you. You can use the context parameter <code>"useConcurrentLocks": true</code> for individual tasks and datasources or enable concurrent append and replace at a cluster level using <code>druid.indexer.task.default.context</code>.</p> |
| <p><a href="https://github.com/apache/druid/pull/15684" target="_blank" rel="noopener noreferrer">#15684</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="enabled-empty-ingest-queries">Enabled empty ingest queries<a href="#enabled-empty-ingest-queries" class="hash-link" aria-label="Direct link to Enabled empty ingest queries" title="Direct link to Enabled empty ingest queries"></a></h4> |
| <p>The MSQ task engine now allows empty ingest queries by default. For queries that don't generate any output rows, the MSQ task engine reports zero values for <code>numTotalRows</code> and <code>totalSizeInBytes</code> instead of null. Previously, ingest queries that produced no data would fail with the <code>InsertCannotBeEmpty</code> MSQ fault.</p> |
| <p>To revert to the original behavior, set the MSQ query parameter <code>failOnEmptyInsert</code> to <code>true</code>.</p> |
| <p><a href="https://github.com/apache/druid/pull/15495" target="_blank" rel="noopener noreferrer">#15495</a> <a href="https://github.com/apache/druid/pull/15674" target="_blank" rel="noopener noreferrer">#15674</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="enabled-query-request-queuing-by-default-when-total-laning-is-turned-on">Enabled query request queuing by default when total laning is turned on<a href="#enabled-query-request-queuing-by-default-when-total-laning-is-turned-on" class="hash-link" aria-label="Direct link to Enabled query request queuing by default when total laning is turned on" title="Direct link to Enabled query request queuing by default when total laning is turned on"></a></h4> |
| <p>When query scheduler threads are less than server HTTP threads, total laning turns on. |
| This reserves some HTTP threads for non-query requests such as health checks. |
| The total laning previously would reject any query request that exceeds the lane capacity. |
| Now, excess requests will instead be queued with a timeout equal to <code>MIN(Integer.MAX_VALUE, druid.server.http.maxQueryTimeout)</code>.</p> |
| <p><a href="https://github.com/apache/druid/pull/15440" target="_blank" rel="noopener noreferrer">#15440</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="changed-how-empty-or-null-array-columns-are-stored">Changed how empty or null array columns are stored<a href="#changed-how-empty-or-null-array-columns-are-stored" class="hash-link" aria-label="Direct link to Changed how empty or null array columns are stored" title="Direct link to Changed how empty or null array columns are stored"></a></h4> |
| <p>Columns ingested with the auto column indexer that contain only empty or null arrays are now stored as <code>ARRAY\<LONG\></code> instead of <code>COMPLEX<json\></code>.</p> |
| <p><a href="https://github.com/apache/druid/pull/15505" target="_blank" rel="noopener noreferrer">#15505</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="changed-how-druid-allocates-weekly-segments">Changed how Druid allocates weekly segments<a href="#changed-how-druid-allocates-weekly-segments" class="hash-link" aria-label="Direct link to Changed how Druid allocates weekly segments" title="Direct link to Changed how Druid allocates weekly segments"></a></h4> |
| <p>When the requested granularity is a month or larger but a segment can't be allocated, Druid resorts to day partitioning. |
| Unless explicitly specified, Druid skips week-granularity segments for data partitioning because these segments don't align with the end of the month or more coarse-grained intervals.</p> |
| <p>Previously, if Druid couldn't allocate segments by month, it tried allocating them by week next. |
| In the new behavior, Druid skips partitioning by week and goes directly to day. Week segments can only be allocated if the chosen partitioning in the append task is WEEK.</p> |
| <p><a href="https://github.com/apache/druid/pull/15589" target="_blank" rel="noopener noreferrer">#15589</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="removed-the-auto-search-strategy">Removed the <code>auto</code> search strategy<a href="#removed-the-auto-search-strategy" class="hash-link" aria-label="Direct link to removed-the-auto-search-strategy" title="Direct link to removed-the-auto-search-strategy"></a></h4> |
| <p>Removed the <code>auto</code> search strategy from the native search query. Setting <code>searchStrategy</code> to <code>auto</code> is now equivalent to <code>useIndexes</code>.</p> |
| <p><a href="https://github.com/apache/druid/pull/15550" target="_blank" rel="noopener noreferrer">#15550</a></p> |
| <h2 class="anchor anchorWithStickyNavbar_LWe7" id="2800">28.0.0<a href="#2800" class="hash-link" aria-label="Direct link to 28.0.0" title="Direct link to 28.0.0"></a></h2> |
| <h3 class="anchor anchorWithStickyNavbar_LWe7" id="upgrade-notes-4">Upgrade notes<a href="#upgrade-notes-4" class="hash-link" aria-label="Direct link to Upgrade notes" title="Direct link to Upgrade notes"></a></h3> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="upgrade-druid-segments-table">Upgrade Druid segments table<a href="#upgrade-druid-segments-table" class="hash-link" aria-label="Direct link to Upgrade Druid segments table" title="Direct link to Upgrade Druid segments table"></a></h4> |
| <p>Druid 28.0.0 adds a new column to the Druid metadata table that requires an update to the table.</p> |
| <p>If <code>druid.metadata.storage.connector.createTables</code> is set to <code>true</code> and the metadata store user has DDL privileges, the segments table gets automatically updated at startup to include the new <code>used_status_last_updated</code> column. No additional work is needed for the upgrade.</p> |
| <p>If either of those requirements are not met, pre-upgrade steps are required. You must make these updates before you upgrade to Druid 28.0.0, or the Coordinator and Overlord processes fail.</p> |
| <p>Although you can manually alter your table to add the new <code>used_status_last_updated</code> column, Druid also provides a <a href="https://druid.apache.org/docs/latest/operations/metadata-migration/#create-druid-tables" target="_blank" rel="noopener noreferrer">CLI tool</a> to do it.</p> |
| <p><a href="https://github.com/apache/druid/pull/12599" target="_blank" rel="noopener noreferrer">#12599</a> <a href="https://github.com/apache/druid/pull/14868" target="_blank" rel="noopener noreferrer">#14868</a></p> |
| <p>In the example commands below:</p> |
| <ul> |
| <li><code>lib</code> is the Druid lib directory</li> |
| <li><code>extensions</code> is the Druid extensions directory</li> |
| <li><code>base</code> corresponds to the value of <code>druid.metadata.storage.tables.base</code> in the configuration, <code>druid</code> by default.</li> |
| <li>The <code>--connectURI</code> parameter corresponds to the value of <code>druid.metadata.storage.connector.connectURI</code>.</li> |
| <li>The <code>--user</code> parameter corresponds to the value of <code>druid.metadata.storage.connector.user</code>.</li> |
| <li>The <code>--password</code> parameter corresponds to the value of <code>druid.metadata.storage.connector.password</code>.</li> |
| <li>The <code>--action</code> parameter corresponds to the update action you are executing. In this case, it is <code>add-last-used-to-segments</code></li> |
| </ul> |
| <h5 class="anchor anchorWithStickyNavbar_LWe7" id="upgrade-step-for-mysql">Upgrade step for MySQL<a href="#upgrade-step-for-mysql" class="hash-link" aria-label="Direct link to Upgrade step for MySQL" title="Direct link to Upgrade step for MySQL"></a></h5> |
| <div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#bfc7d5;background-color:#292d3e"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">cd ${DRUID_ROOT}</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">java -classpath "lib/*" -Dlog4j.configurationFile=conf/druid/cluster/_common/log4j2.xml -Ddruid.extensions.directory="extensions" -Ddruid.extensions.loadList=[\"mysql-metadata-storage\"] -Ddruid.metadata.storage.type=mysql org.apache.druid.cli.Main tools metadata-update --connectURI="<mysql-uri>" --user USER --password PASSWORD --base druid --action add-used-flag-last-updated-to-segments</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div> |
| <h5 class="anchor anchorWithStickyNavbar_LWe7" id="upgrade-step-for-postgresql">Upgrade step for PostgreSQL<a href="#upgrade-step-for-postgresql" class="hash-link" aria-label="Direct link to Upgrade step for PostgreSQL" title="Direct link to Upgrade step for PostgreSQL"></a></h5> |
| <div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#bfc7d5;background-color:#292d3e"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">cd ${DRUID_ROOT}</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">java -classpath "lib/*" -Dlog4j.configurationFile=conf/druid/cluster/_common/log4j2.xml -Ddruid.extensions.directory="extensions" -Ddruid.extensions.loadList=[\"postgresql-metadata-storage\"] -Ddruid.metadata.storage.type=postgresql org.apache.druid.cli.Main tools metadata-update --connectURI="<postgresql-uri>" --user USER --password PASSWORD --base druid --action add-used-flag-last-updated-to-segments</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div> |
| <h5 class="anchor anchorWithStickyNavbar_LWe7" id="manual-upgrade-step">Manual upgrade step<a href="#manual-upgrade-step" class="hash-link" aria-label="Direct link to Manual upgrade step" title="Direct link to Manual upgrade step"></a></h5> |
| <div class="language-SQL language-sql codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sql codeBlock_bY9V thin-scrollbar" style="color:#bfc7d5;background-color:#292d3e"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token keyword" style="font-style:italic">ALTER</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">TABLE</span><span class="token plain"> druid_segments</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token keyword" style="font-style:italic">ADD</span><span class="token plain"> used_status_last_updated </span><span class="token keyword" style="font-style:italic">varchar</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token number" style="color:rgb(247, 140, 108)">255</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token punctuation" style="color:rgb(199, 146, 234)">;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="recommended-syntax-for-sql-unnest">Recommended syntax for SQL UNNEST<a href="#recommended-syntax-for-sql-unnest" class="hash-link" aria-label="Direct link to Recommended syntax for SQL UNNEST" title="Direct link to Recommended syntax for SQL UNNEST"></a></h4> |
| <p>The recommended syntax for SQL UNNEST has changed. We recommend using CROSS JOIN instead of commas for most queries to prevent issues with precedence. For example, use:</p> |
| <div class="language-sql codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sql codeBlock_bY9V thin-scrollbar" style="color:#bfc7d5;background-color:#292d3e"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token keyword" style="font-style:italic">SELECT</span><span class="token plain"> column_alias_name1 </span><span class="token keyword" style="font-style:italic">FROM</span><span class="token plain"> datasource </span><span class="token keyword" style="font-style:italic">CROSS</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">JOIN</span><span class="token plain"> UNNEST</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">source_expression1</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">AS</span><span class="token plain"> table_alias_name1</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">column_alias_name1</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">CROSS</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">JOIN</span><span class="token plain"> UNNEST</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">source_expression2</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">AS</span><span class="token plain"> table_alias_name2</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">column_alias_name2</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div> |
| <p>Do not use:</p> |
| <div class="language-sql codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sql codeBlock_bY9V thin-scrollbar" style="color:#bfc7d5;background-color:#292d3e"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token keyword" style="font-style:italic">SELECT</span><span class="token plain"> column_alias_name </span><span class="token keyword" style="font-style:italic">FROM</span><span class="token plain"> datasource</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> UNNEST</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">source_expression1</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">AS</span><span class="token plain"> table_alias_name1</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">column_alias_name1</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> UNNEST</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">source_expression2</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">AS</span><span class="token plain"> table_alias_name2</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">column_alias_name2</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="dynamic-parameters">Dynamic parameters<a href="#dynamic-parameters" class="hash-link" aria-label="Direct link to Dynamic parameters" title="Direct link to Dynamic parameters"></a></h4> |
| <p>The Apache Calcite version has been upgraded from 1.21 to 1.35. As part of the Calcite upgrade, the behavior of type inference for dynamic parameters has changed. To avoid any type interference issues, explicitly <code>CAST</code> all dynamic parameters as a specific data type in SQL queries. For example, use:</p> |
| <div class="language-sql codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sql codeBlock_bY9V thin-scrollbar" style="color:#bfc7d5;background-color:#292d3e"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token keyword" style="font-style:italic">SELECT</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token plain"> </span><span class="token operator" style="color:rgb(137, 221, 255)">*</span><span class="token plain"> CAST </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">? </span><span class="token keyword" style="font-style:italic">as</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">DOUBLE</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token operator" style="color:rgb(137, 221, 255)">/</span><span class="token number" style="color:rgb(247, 140, 108)">2</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">as</span><span class="token plain"> tmp</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div> |
| <p>Do not use:</p> |
| <div class="language-sql codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sql codeBlock_bY9V thin-scrollbar" style="color:#bfc7d5;background-color:#292d3e"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token keyword" style="font-style:italic">SELECT</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token plain"> </span><span class="token operator" style="color:rgb(137, 221, 255)">*</span><span class="token plain"> ?</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token operator" style="color:rgb(137, 221, 255)">/</span><span class="token number" style="color:rgb(247, 140, 108)">2</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">as</span><span class="token plain"> tmp</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="nested-column-format">Nested column format<a href="#nested-column-format" class="hash-link" aria-label="Direct link to Nested column format" title="Direct link to Nested column format"></a></h4> |
| <p><code>json</code> type columns created with Druid 28.0.0 are not backwards compatible with Druid versions older than 26.0.0. |
| If you are upgrading from a version prior to Druid 26.0.0 and you use <code>json</code> columns, upgrade to Druid 26.0.0 before you upgrade to Druid 28.0.0. |
| Additionally, to downgrade to a version older than Druid 26.0.0, any new segments created in Druid 28.0.0 should be re-ingested using Druid 26.0.0 or 27.0.0 prior to further downgrading.</p> |
| <p>When upgrading from a previous version, you can continue to write nested columns in a backwards compatible format (version 4).</p> |
| <p>In a classic batch ingestion job, include <code>formatVersion</code> in the <code>dimensions</code> list of the <code>dimensionsSpec</code> property. For example:</p> |
| <div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar" style="color:#bfc7d5;background-color:#292d3e"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"dimensionsSpec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"dimensions"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"product"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"department"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"json"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"name"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"shipTo"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"formatVersion"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">4</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div> |
| <p>To set the default nested column version, set the desired format version in the common runtime properties. For example:</p> |
| <div class="language-java codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-java codeBlock_bY9V thin-scrollbar" style="color:#bfc7d5;background-color:#292d3e"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.indexing.formats.nestedColumnFormatVersion=4</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="sql-compatibility">SQL compatibility<a href="#sql-compatibility" class="hash-link" aria-label="Direct link to SQL compatibility" title="Direct link to SQL compatibility"></a></h4> |
| <p>Starting with Druid 28.0.0, the default way Druid treats nulls and booleans has changed.</p> |
| <p>For nulls, Druid now differentiates between an empty string and a record with no data as well as between an empty numerical record and <code>0</code>.<br> |
| <!-- -->You can revert to the previous behavior by setting <code>druid.generic.useDefaultValueForNull</code> to <code>true</code>.</p> |
| <p>This property affects both storage and querying, and must be set on all Druid service types to be available at both ingestion time and query time. Reverting this setting to the old value restores the previous behavior without reingestion.</p> |
| <p>For booleans, Druid now strictly uses <code>1</code> (true) or <code>0</code> (false). Previously, true and false could be represented either as <code>true</code> and <code>false</code> as well as <code>1</code> and <code>0</code>, respectively. In addition, Druid now returns a null value for boolean comparisons like <code>True && NULL</code>.</p> |
| <p>You can revert to the previous behavior by setting <code>druid.expressions.useStrictBooleans</code> to <code>false</code>. |
| This property affects both storage and querying, and must be set on all Druid service types to be available at both ingestion time and query time. Reverting this setting to the old value restores the previous behavior without reingestion.</p> |
| <p>The following table illustrates some example scenarios and the impact of the changes.</p> |
| <details class="details_lb9f alert alert--info details_b_Ee" data-collapsed="true"><summary>Show the table</summary><div><div class="collapsibleContent_i85q"><table><thead><tr><th>Query</th><th>Druid 27.0.0 and earlier</th><th>Druid 28.0.0 and later</th></tr></thead><tbody><tr><td>Query empty string</td><td>Empty string (<code>''</code>) or null</td><td>Empty string (<code>''</code>)</td></tr><tr><td>Query null string</td><td>Null or empty</td><td>Null</td></tr><tr><td>COUNT(*)</td><td>All rows, including nulls</td><td>All rows, including nulls</td></tr><tr><td>COUNT(column)</td><td>All rows excluding empty strings</td><td>All rows including empty strings but excluding nulls</td></tr><tr><td>Expression 100 && 11</td><td>11</td><td>1</td></tr><tr><td>Expression 100 || 11</td><td>100</td><td>1</td></tr><tr><td>Null FLOAT/DOUBLE column</td><td>0.0</td><td>Null</td></tr><tr><td>Null LONG column</td><td>0</td><td>Null</td></tr><tr><td>Null <code>__time</code> column</td><td>0, meaning 1970-01-01 00:00:00 UTC</td><td>1970-01-01 00:00:00 UTC</td></tr><tr><td>Null MVD column</td><td><code>''</code></td><td>Null</td></tr><tr><td>ARRAY</td><td>Null</td><td>Null</td></tr><tr><td>COMPLEX</td><td>none</td><td>Null</td></tr></tbody></table></div></div></details> |
| <p>Before upgrading to Druid 28.0.0, update your queries to account for the changed behavior as described in the following sections.</p> |
| <h5 class="anchor anchorWithStickyNavbar_LWe7" id="null-filters">NULL filters<a href="#null-filters" class="hash-link" aria-label="Direct link to NULL filters" title="Direct link to NULL filters"></a></h5> |
| <p>If your queries use NULL in the filter condition to match both nulls and empty strings, you should add an explicit filter clause for empty strings. For example, update <code>s IS NULL</code> to <code>s IS NULL OR s = ''</code>.</p> |
| <h5 class="anchor anchorWithStickyNavbar_LWe7" id="count-functions">COUNT functions<a href="#count-functions" class="hash-link" aria-label="Direct link to COUNT functions" title="Direct link to COUNT functions"></a></h5> |
| <p><code>COUNT(column)</code> now counts empty strings. If you want to continue excluding empty strings from the count, replace <code>COUNT(column)</code> with <code>COUNT(column) FILTER(WHERE column <> '')</code>.</p> |
| <h5 class="anchor anchorWithStickyNavbar_LWe7" id="groupby-queries">GroupBy queries<a href="#groupby-queries" class="hash-link" aria-label="Direct link to GroupBy queries" title="Direct link to GroupBy queries"></a></h5> |
| <p>GroupBy queries on columns containing null values can now have additional entries as nulls can co-exist with empty strings.</p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="stop-supervisors-that-ingest-from-multiple-kafka-topics-before-downgrading">Stop Supervisors that ingest from multiple Kafka topics before downgrading<a href="#stop-supervisors-that-ingest-from-multiple-kafka-topics-before-downgrading" class="hash-link" aria-label="Direct link to Stop Supervisors that ingest from multiple Kafka topics before downgrading" title="Direct link to Stop Supervisors that ingest from multiple Kafka topics before downgrading"></a></h4> |
| <p>If you have added supervisors that ingest from multiple Kafka topics in Druid 28.0.0 or later, stop those supervisors before downgrading to a version prior to Druid 28.0.0 because the supervisors will fail in versions prior to Druid 28.0.0.</p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="lenientaggregatormerge-deprecated"><code>lenientAggregatorMerge</code> deprecated<a href="#lenientaggregatormerge-deprecated" class="hash-link" aria-label="Direct link to lenientaggregatormerge-deprecated" title="Direct link to lenientaggregatormerge-deprecated"></a></h4> |
| <p><code>lenientAggregatorMerge</code> property in segment metadata queries has been deprecated. It will be removed in future releases. |
| Use <code>aggregatorMergeStrategy</code> instead. <code>aggregatorMergeStrategy</code> also supports the <code>latest</code> and <code>earliest</code> strategies in addition to <code>strict</code> and <code>lenient</code> strategies from <code>lenientAggregatorMerge</code>.</p> |
| <p><a href="https://github.com/apache/druid/pull/14560" target="_blank" rel="noopener noreferrer">#14560</a> |
| <a href="https://github.com/apache/druid/pull/14598" target="_blank" rel="noopener noreferrer">#14598</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="broker-parallel-merge-config-options">Broker parallel merge config options<a href="#broker-parallel-merge-config-options" class="hash-link" aria-label="Direct link to Broker parallel merge config options" title="Direct link to Broker parallel merge config options"></a></h4> |
| <p>The paths for <code>druid.processing.merge.pool.*</code> and <code>druid.processing.merge.task.*</code> have been flattened to use <code>druid.processing.merge.*</code> instead. The legacy paths for the configs are now deprecated and will be removed in a future release. Migrate your settings to use the new paths because the old paths will be ignored in the future.</p> |
| <p><a href="https://github.com/apache/druid/pull/14695" target="_blank" rel="noopener noreferrer">#14695</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="ingestion-options-for-array-typed-columns">Ingestion options for ARRAY typed columns<a href="#ingestion-options-for-array-typed-columns" class="hash-link" aria-label="Direct link to Ingestion options for ARRAY typed columns" title="Direct link to Ingestion options for ARRAY typed columns"></a></h4> |
| <p>Starting with Druid 28.0.0, the MSQ task engine can detect and ingest arrays as ARRAY typed columns when you set the query context parameter <code>arrayIngestMode</code> to <code>array</code>. |
| The <code>arrayIngestMode</code> context parameter controls how ARRAY type values are stored in Druid segments.</p> |
| <p>When you set <code>arrayIngestMode</code> to <code>array</code> (recommended for SQL compliance), the MSQ task engine stores all ARRAY typed values in <a href="https://druid.apache.org/docs/latest/querying/arrays" target="_blank" rel="noopener noreferrer">ARRAY typed columns</a> and supports storing both VARCHAR and numeric typed arrays.</p> |
| <p>For backwards compatibility, <code>arrayIngestMode</code> defaults to <code>mvd</code>. When <code>"arrayIngestMode":"mvd"</code>, Druid only supports VARCHAR typed arrays and stores them as <a href="https://druid.apache.org/docs/latest/querying/multi-value-dimensions" target="_blank" rel="noopener noreferrer">multi-value string columns</a>.</p> |
| <p>When you set <code>arrayIngestMode</code> to <code>none</code>, Druid throws an exception when trying to store any type of arrays.</p> |
| <p>For more information on how to ingest <code>ARRAY</code> typed columns with SQL-based ingestion, see <a href="https://druid.apache.org/docs/latest/querying/sql-data-types#arrays" target="_blank" rel="noopener noreferrer">SQL data types</a> and <a href="https://druid.apache.org/docs/latest/querying/arrays" target="_blank" rel="noopener noreferrer">Array columns</a>.</p> |
| <h3 class="anchor anchorWithStickyNavbar_LWe7" id="incompatible-changes-4">Incompatible changes<a href="#incompatible-changes-4" class="hash-link" aria-label="Direct link to Incompatible changes" title="Direct link to Incompatible changes"></a></h3> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="removed-hadoop-2">Removed Hadoop 2<a href="#removed-hadoop-2" class="hash-link" aria-label="Direct link to Removed Hadoop 2" title="Direct link to Removed Hadoop 2"></a></h4> |
| <p>Support for Hadoop 2 has been removed. |
| Migrate to SQL-based ingestion or JSON-based batch ingestion if you are using Hadoop 2.x for ingestion today. |
| If migrating to Druid's built-in ingestion is not possible, you must upgrade your Hadoop infrastructure to 3.x+ before upgrading to Druid 28.0.0.</p> |
| <p><a href="https://github.com/apache/druid/pull/14763" target="_blank" rel="noopener noreferrer">#14763</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="removed-groupby-v1">Removed GroupBy v1<a href="#removed-groupby-v1" class="hash-link" aria-label="Direct link to Removed GroupBy v1" title="Direct link to Removed GroupBy v1"></a></h4> |
| <p>The GroupBy v1 engine has been removed. Use the GroupBy v2 engine instead, which has been the default GroupBy engine for several releases. |
| There should be no impact on your queries.</p> |
| <p>Additionally, <code>AggregatorFactory.getRequiredColumns</code> has been deprecated and will be removed in a future release. If you have an extension that implements <code>AggregatorFactory</code>, then this method should be removed from your implementation.</p> |
| <p><a href="https://github.com/apache/druid/pull/14866" target="_blank" rel="noopener noreferrer">#14866</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="removed-coordinator-dynamic-configs">Removed Coordinator dynamic configs<a href="#removed-coordinator-dynamic-configs" class="hash-link" aria-label="Direct link to Removed Coordinator dynamic configs" title="Direct link to Removed Coordinator dynamic configs"></a></h4> |
| <p>The <code>decommissioningMaxPercentOfMaxSegmentsToMove</code> config has been removed. |
| The use case for this config is handled by smart segment loading now, which is enabled by default.</p> |
| <p><a href="https://github.com/apache/druid/pull/14923" target="_blank" rel="noopener noreferrer">#14923</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="removed-cachingcost-strategy">Removed <code>cachingCost</code> strategy<a href="#removed-cachingcost-strategy" class="hash-link" aria-label="Direct link to removed-cachingcost-strategy" title="Direct link to removed-cachingcost-strategy"></a></h4> |
| <p>The <code>cachingCost</code> strategy for segment loading has been removed. |
| Use <code>cost</code> instead, which has the same benefits as <code>cachingCost</code>.</p> |
| <p>If you have <code>cachingCost</code> set, the system ignores this setting and automatically uses <code>cost</code>.</p> |
| <p><a href="https://github.com/apache/druid/pull/14798" target="_blank" rel="noopener noreferrer">#14798</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="removed-insertcannotorderbydescending">Removed <code>InsertCannotOrderByDescending</code><a href="#removed-insertcannotorderbydescending" class="hash-link" aria-label="Direct link to removed-insertcannotorderbydescending" title="Direct link to removed-insertcannotorderbydescending"></a></h4> |
| <p>The deprecated MSQ fault <code>InsertCannotOrderByDescending</code> has been removed.</p> |
| <p><a href="https://github.com/apache/druid/pull/14588" target="_blank" rel="noopener noreferrer">#14588</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="removed-the-backward-compatibility-code-for-the-handoff-api">Removed the backward compatibility code for the Handoff API<a href="#removed-the-backward-compatibility-code-for-the-handoff-api" class="hash-link" aria-label="Direct link to Removed the backward compatibility code for the Handoff API" title="Direct link to Removed the backward compatibility code for the Handoff API"></a></h4> |
| <p>The backward compatibility code for the Handoff API in <code>CoordinatorBasedSegmentHandoffNotifier</code> has been removed. |
| If you are upgrading from a Druid version older than 0.14.0, upgrade to a newer version of Druid before upgrading to Druid 28.0.0.</p> |
| <p><a href="https://github.com/apache/druid/pull/14652" target="_blank" rel="noopener noreferrer">#14652</a></p> |
| <h2 class="anchor anchorWithStickyNavbar_LWe7" id="2700">27.0.0<a href="#2700" class="hash-link" aria-label="Direct link to 27.0.0" title="Direct link to 27.0.0"></a></h2> |
| <h3 class="anchor anchorWithStickyNavbar_LWe7" id="upgrade-notes-5">Upgrade notes<a href="#upgrade-notes-5" class="hash-link" aria-label="Direct link to Upgrade notes" title="Direct link to Upgrade notes"></a></h3> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="worker-input-bytes-for-sql-based-ingestion">Worker input bytes for SQL-based ingestion<a href="#worker-input-bytes-for-sql-based-ingestion" class="hash-link" aria-label="Direct link to Worker input bytes for SQL-based ingestion" title="Direct link to Worker input bytes for SQL-based ingestion"></a></h4> |
| <p>The maximum input bytes for each worker for SQL-based ingestion is now 512 MiB (previously 10 GiB).</p> |
| <p><a href="https://github.com/apache/druid/pull/14307" target="_blank" rel="noopener noreferrer">#14307</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="parameter-execution-changes-for-kafka">Parameter execution changes for Kafka<a href="#parameter-execution-changes-for-kafka" class="hash-link" aria-label="Direct link to Parameter execution changes for Kafka" title="Direct link to Parameter execution changes for Kafka"></a></h4> |
| <p>When using the built-in <code>FileConfigProvider</code> for Kafka, interpolations are now intercepted by the JsonConfigurator instead of being passed down to the Kafka provider. This breaks existing deployments.</p> |
| <p>For more information, see <a href="https://cwiki.apache.org/confluence/display/KAFKA/KIP-297%3A+Externalizing+Secrets+for+Connect+Configurations" target="_blank" rel="noopener noreferrer">KIP-297</a>.</p> |
| <p><a href="https://github.com/apache/druid/pull/13023" target="_blank" rel="noopener noreferrer">#13023</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="hadoop-2-deprecated">Hadoop 2 deprecated<a href="#hadoop-2-deprecated" class="hash-link" aria-label="Direct link to Hadoop 2 deprecated" title="Direct link to Hadoop 2 deprecated"></a></h4> |
| <p>Many of the important dependent libraries that Druid uses no longer support Hadoop 2. In order for Druid to stay current and have pathways to mitigate security vulnerabilities, the community has decided to deprecate support for Hadoop 2.x releases starting this release. Starting with Druid 28.x, Hadoop 3.x is the only supported Hadoop version.</p> |
| <p>Consider migrating to SQL-based ingestion or native ingestion if you are using Hadoop 2.x for ingestion today. If migrating to Druid ingestion is not possible, plan to upgrade your Hadoop infrastructure before upgrading to the next Druid release.</p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="groupby-v1-deprecated">GroupBy v1 deprecated<a href="#groupby-v1-deprecated" class="hash-link" aria-label="Direct link to GroupBy v1 deprecated" title="Direct link to GroupBy v1 deprecated"></a></h4> |
| <p>GroupBy queries using the v1 legacy engine has been deprecated. It will be removed in future releases. Use v2 instead. Note that v2 has been the default GroupBy engine.</p> |
| <p>For more information, see <a href="https://druid.apache.org/docs/latest/querying/groupbyquery.html" target="_blank" rel="noopener noreferrer">GroupBy queries</a>.</p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="push-based-real-time-ingestion-deprecated">Push-based real-time ingestion deprecated<a href="#push-based-real-time-ingestion-deprecated" class="hash-link" aria-label="Direct link to Push-based real-time ingestion deprecated" title="Direct link to Push-based real-time ingestion deprecated"></a></h4> |
| <p>Support for push-based real-time ingestion has been deprecated. It will be removed in future releases.</p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="cachingcost-segment-balancing-strategy-deprecated"><code>cachingCost</code> segment balancing strategy deprecated<a href="#cachingcost-segment-balancing-strategy-deprecated" class="hash-link" aria-label="Direct link to cachingcost-segment-balancing-strategy-deprecated" title="Direct link to cachingcost-segment-balancing-strategy-deprecated"></a></h4> |
| <p>The <code>cachingCost</code> strategy has been deprecated and will be removed in future releases. Use an alternate segment balancing strategy instead, such as <code>cost</code>.</p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="segment-loading-config-changes">Segment loading config changes<a href="#segment-loading-config-changes" class="hash-link" aria-label="Direct link to Segment loading config changes" title="Direct link to Segment loading config changes"></a></h4> |
| <p>The following segment related configs are now deprecated and will be removed in future releases:</p> |
| <ul> |
| <li><code>maxSegmentsInNodeLoadingQueue</code></li> |
| <li><code>maxSegmentsToMove</code></li> |
| <li><code>replicationThrottleLimit</code></li> |
| <li><code>useRoundRobinSegmentAssignment</code></li> |
| <li><code>replicantLifetime</code></li> |
| <li><code>maxNonPrimaryReplicantsToLoad</code></li> |
| <li><code>decommissioningMaxPercentOfMaxSegmentsToMove</code></li> |
| </ul> |
| <p>Use <code>smartSegmentLoading</code> mode instead, which calculates values for these variables automatically.</p> |
| <p>Additionally, the defaults for the following Coordinator dynamic configs have changed:</p> |
| <ul> |
| <li><code>maxsegmentsInNodeLoadingQueue</code> : 500, previously 100</li> |
| <li><code>maxSegmentsToMove</code>: 100, previously 5</li> |
| <li><code>replicationThrottleLimit</code>: 500, previously 10</li> |
| </ul> |
| <p>These new defaults can improve performance for most use cases.</p> |
| <p><a href="https://github.com/apache/druid/pull/13197" target="_blank" rel="noopener noreferrer">#13197</a> |
| <a href="https://github.com/apache/druid/pull/14269" target="_blank" rel="noopener noreferrer">#14269</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="sysmonitor-support-deprecated"><code>SysMonitor</code> support deprecated<a href="#sysmonitor-support-deprecated" class="hash-link" aria-label="Direct link to sysmonitor-support-deprecated" title="Direct link to sysmonitor-support-deprecated"></a></h4> |
| <p>Switch to <code>OshiSysMonitor</code> as <code>SysMonitor</code> is now deprecated and will be removed in future releases.</p> |
| <h3 class="anchor anchorWithStickyNavbar_LWe7" id="incompatible-changes-5">Incompatible changes<a href="#incompatible-changes-5" class="hash-link" aria-label="Direct link to Incompatible changes" title="Direct link to Incompatible changes"></a></h3> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="removed-property-for-setting-max-bytes-for-dimension-lookup-cache">Removed property for setting max bytes for dimension lookup cache<a href="#removed-property-for-setting-max-bytes-for-dimension-lookup-cache" class="hash-link" aria-label="Direct link to Removed property for setting max bytes for dimension lookup cache" title="Direct link to Removed property for setting max bytes for dimension lookup cache"></a></h4> |
| <p><code>druid.processing.columnCache.sizeBytes</code> has been removed since it provided limited utility after a number of internal changes. Leaving this config is harmless, but it does nothing.</p> |
| <p><a href="https://github.com/apache/druid/pull/14500" target="_blank" rel="noopener noreferrer">#14500</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="removed-coordinator-dynamic-configs-1">Removed Coordinator dynamic configs<a href="#removed-coordinator-dynamic-configs-1" class="hash-link" aria-label="Direct link to Removed Coordinator dynamic configs" title="Direct link to Removed Coordinator dynamic configs"></a></h4> |
| <p>The following Coordinator dynamic configs have been removed:</p> |
| <ul> |
| <li><code>emitBalancingStats</code>: Stats for errors encountered while balancing will always be emitted. Other debugging stats will not be emitted but can be logged by setting the appropriate <code>debugDimensions</code>.</li> |
| <li><code>useBatchedSegmentSampler</code> and <code>percentOfSegmentsToConsiderPerMove</code>: Batched segment sampling is now the standard and will always be on.</li> |
| </ul> |
| <p>Use the new <a href="https://druid.apache.org/docs/latest/configuration/#smart-segment-loading" target="_blank" rel="noopener noreferrer">smart segment loading</a> mode instead.</p> |
| <p><a href="https://github.com/apache/druid/pull/14524" target="_blank" rel="noopener noreferrer">#14524</a></p> |
| <h2 class="anchor anchorWithStickyNavbar_LWe7" id="2600">26.0.0<a href="#2600" class="hash-link" aria-label="Direct link to 26.0.0" title="Direct link to 26.0.0"></a></h2> |
| <h3 class="anchor anchorWithStickyNavbar_LWe7" id="upgrade-notes-6">Upgrade notes<a href="#upgrade-notes-6" class="hash-link" aria-label="Direct link to Upgrade notes" title="Direct link to Upgrade notes"></a></h3> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="real-time-tasks">Real-time tasks<a href="#real-time-tasks" class="hash-link" aria-label="Direct link to Real-time tasks" title="Direct link to Real-time tasks"></a></h4> |
| <p>Optimized query performance by lowering the default maxRowsInMemory for real-time ingestion, which might lower overall ingestion throughput.</p> |
| <p><a href="https://github.com/apache/druid/pull/13939" target="_blank" rel="noopener noreferrer">#13939</a></p> |
| <h3 class="anchor anchorWithStickyNavbar_LWe7" id="incompatible-changes-6">Incompatible changes<a href="#incompatible-changes-6" class="hash-link" aria-label="Direct link to Incompatible changes" title="Direct link to Incompatible changes"></a></h3> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="firehose-ingestion-removed">Firehose ingestion removed<a href="#firehose-ingestion-removed" class="hash-link" aria-label="Direct link to Firehose ingestion removed" title="Direct link to Firehose ingestion removed"></a></h4> |
| <p>The firehose/parser specification used by legacy Druid streaming formats is removed. |
| Firehose ingestion was deprecated in version 0.17, and support for this ingestion was removed in version 24.0.0.</p> |
| <p><a href="https://github.com/apache/druid/pull/12852" target="_blank" rel="noopener noreferrer">#12852</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="information-schema-now-uses-numeric-column-types">Information schema now uses numeric column types<a href="#information-schema-now-uses-numeric-column-types" class="hash-link" aria-label="Direct link to Information schema now uses numeric column types" title="Direct link to Information schema now uses numeric column types"></a></h4> |
| <p>The Druid system table (<code>INFORMATION_SCHEMA</code>) now uses SQL types instead of Druid types for columns. This change makes the <code>INFORMATION_SCHEMA</code> table behave more like standard SQL. You may need to update your queries in the following scenarios in order to avoid unexpected results if you depend either of the following:</p> |
| <ul> |
| <li>Numeric fields being treated as strings.</li> |
| <li>Column numbering starting at 0. Column numbering is now 1-based.</li> |
| </ul> |
| <p><a href="https://github.com/apache/druid/pull/13777" target="_blank" rel="noopener noreferrer">#13777</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="frontcoded-segment-format-change"><code>frontCoded</code> segment format change<a href="#frontcoded-segment-format-change" class="hash-link" aria-label="Direct link to frontcoded-segment-format-change" title="Direct link to frontcoded-segment-format-change"></a></h4> |
| <p>The <code>frontCoded</code> type of <code>stringEncodingStrategy</code> on <code>indexSpec</code> with a new segment format version, which typically has faster read speeds and reduced segment size. This improvement is backwards incompatible with Druid 25.0.0.</p> |
| <h2 class="anchor anchorWithStickyNavbar_LWe7" id="2500">25.0.0<a href="#2500" class="hash-link" aria-label="Direct link to 25.0.0" title="Direct link to 25.0.0"></a></h2> |
| <h3 class="anchor anchorWithStickyNavbar_LWe7" id="upgrade-notes-7">Upgrade notes<a href="#upgrade-notes-7" class="hash-link" aria-label="Direct link to Upgrade notes" title="Direct link to Upgrade notes"></a></h3> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="default-http-based-segment-discovery-and-task-management">Default HTTP-based segment discovery and task management<a href="#default-http-based-segment-discovery-and-task-management" class="hash-link" aria-label="Direct link to Default HTTP-based segment discovery and task management" title="Direct link to Default HTTP-based segment discovery and task management"></a></h4> |
| <p>The default segment discovery method now uses HTTP instead of ZooKeeper.</p> |
| <p>This update changes the defaults for the following properties:</p> |
| <table><thead><tr><th>Property</th><th>New default</th><th>Previous default</th></tr></thead><tbody><tr><td><code>druid.serverview.type</code> for segment management</td><td>http</td><td>batch</td></tr><tr><td><code>druid.coordinator.loadqueuepeon.type</code> for segment management</td><td>http</td><td>curator</td></tr><tr><td><code>druid.indexer.runner.type</code> for the Overlord</td><td>httpRemote</td><td>local</td></tr></tbody></table> |
| <p>To use ZooKeeper instead of HTTP, change the values for the properties back to the previous defaults. ZooKeeper-based implementations for these properties are deprecated and will be removed in a subsequent release.</p> |
| <p><a href="https://github.com/apache/druid/pull/13092" target="_blank" rel="noopener noreferrer">#13092</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="finalizing-hll-and-quantiles-sketch-aggregates">Finalizing HLL and quantiles sketch aggregates<a href="#finalizing-hll-and-quantiles-sketch-aggregates" class="hash-link" aria-label="Direct link to Finalizing HLL and quantiles sketch aggregates" title="Direct link to Finalizing HLL and quantiles sketch aggregates"></a></h4> |
| <p>The aggregation functions for HLL and quantiles sketches returned sketches or numbers when they are finalized depending on where they were in the native query plan.</p> |
| <p>Druid no longer finalizes aggregators in the following two cases:</p> |
| <ul> |
| <li>aggregators appear in the outer level of a query</li> |
| <li>aggregators are used as input to an expression or finalizing-field-access post-aggregator</li> |
| </ul> |
| <p>This change aligns the behavior of HLL and quantiles sketches with theta sketches.</p> |
| <p>To restore old behavior, you can set <code>sqlFinalizeOuterSketches=true</code> in the query context.</p> |
| <p><a href="https://github.com/apache/druid/pull/13247" target="_blank" rel="noopener noreferrer">#13247</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="kill-tasks-mark-segments-as-unused-only-if-specified">Kill tasks mark segments as unused only if specified<a href="#kill-tasks-mark-segments-as-unused-only-if-specified" class="hash-link" aria-label="Direct link to Kill tasks mark segments as unused only if specified" title="Direct link to Kill tasks mark segments as unused only if specified"></a></h4> |
| <p>When you issue a kill task, Druid marks the underlying segments as unused only if explicitly specified. For more information, see the <a href="https://druid.apache.org/docs/latest/api-reference/data-management-api" target="_blank" rel="noopener noreferrer">API reference</a>.</p> |
| <p><a href="https://github.com/apache/druid/pull/13104" target="_blank" rel="noopener noreferrer">#13104</a></p> |
| <h3 class="anchor anchorWithStickyNavbar_LWe7" id="incompatible-changes-7">Incompatible changes<a href="#incompatible-changes-7" class="hash-link" aria-label="Direct link to Incompatible changes" title="Direct link to Incompatible changes"></a></h3> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="upgrade-curator-to-530">Upgrade curator to 5.3.0<a href="#upgrade-curator-to-530" class="hash-link" aria-label="Direct link to Upgrade curator to 5.3.0" title="Direct link to Upgrade curator to 5.3.0"></a></h4> |
| <p>Apache Curator upgraded to the latest version, 5.3.0. This version drops support for ZooKeeper 3.4 but Druid has already officially dropped support in 0.22. In 5.3.0, Curator has removed support for Exhibitor so all related configurations and tests have been removed.</p> |
| <p><a href="https://github.com/apache/druid/pull/12939" target="_blank" rel="noopener noreferrer">#12939</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="fixed-parquet-list-conversion">Fixed Parquet list conversion<a href="#fixed-parquet-list-conversion" class="hash-link" aria-label="Direct link to Fixed Parquet list conversion" title="Direct link to Fixed Parquet list conversion"></a></h4> |
| <p>The behavior of the parquet reader for lists of structured objects has been changed to be consistent with other parquet logical list conversions. The data is now fetched directly, more closely matching its expected structure.</p> |
| <p><a href="https://github.com/apache/druid/pull/13294" target="_blank" rel="noopener noreferrer">#13294</a></p> |
| <h2 class="anchor anchorWithStickyNavbar_LWe7" id="2400">24.0.0<a href="#2400" class="hash-link" aria-label="Direct link to 24.0.0" title="Direct link to 24.0.0"></a></h2> |
| <h3 class="anchor anchorWithStickyNavbar_LWe7" id="upgrade-notes-8">Upgrade notes<a href="#upgrade-notes-8" class="hash-link" aria-label="Direct link to Upgrade notes" title="Direct link to Upgrade notes"></a></h3> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="permissions-for-multi-stage-query-engine">Permissions for multi-stage query engine<a href="#permissions-for-multi-stage-query-engine" class="hash-link" aria-label="Direct link to Permissions for multi-stage query engine" title="Direct link to Permissions for multi-stage query engine"></a></h4> |
| <p>To read external data using the multi-stage query task engine, you must have READ permissions for the EXTERNAL resource type. Users without the correct permission encounter a 403 error when trying to run SQL queries that include EXTERN.</p> |
| <p>The way you assign the permission depends on your authorizer. For example, with <a href="https://github.com/apache/druid/blob/druid-24.0.0/docs/operations/security-user-auth.md" target="_blank" rel="noopener noreferrer">basic security</a> in Druid, add the EXTERNAL READ permission by sending a POST request to the <a href="https://github.com/apache/druid/blob/druid-24.0.0/docs/development/extensions-core/druid-basic-security.md#permissions" target="_blank" rel="noopener noreferrer">roles API</a>.</p> |
| <p>The example adds permissions for users with the admin role using a basic authorizer named MyBasicMetadataAuthorizer. The following permissions are granted:</p> |
| <ul> |
| <li>DATASOURCE READ</li> |
| <li>DATASOURCE WRITE</li> |
| <li>CONFIG READ</li> |
| <li>CONFIG WRITE</li> |
| <li>STATE READ</li> |
| <li>STATE WRITE</li> |
| <li>EXTERNAL READ</li> |
| </ul> |
| <div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#bfc7d5;background-color:#292d3e"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">curl --location --request POST 'http://localhost:8081/druid-ext/basic-security/authorization/db/MyBasicMetadataAuthorizer/roles/admin/permissions' \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--header 'Content-Type: application/json' \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--data-raw '[</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "resource": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "name": ".*",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "type": "DATASOURCE"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "action": "READ"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">},</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "resource": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "name": ".*",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "type": "DATASOURCE"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "action": "WRITE"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">},</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "resource": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "name": ".*",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "type": "CONFIG"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "action": "READ"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">},</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "resource": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "name": ".*",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "type": "CONFIG"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "action": "WRITE"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">},</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "resource": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "name": ".*",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "type": "STATE"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "action": "READ"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">},</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "resource": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "name": ".*",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "type": "STATE"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "action": "WRITE"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">},</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "resource": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "name": "EXTERNAL",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "type": "EXTERNAL"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "action": "READ"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">}</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">]'</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="behavior-for-unused-segments">Behavior for unused segments<a href="#behavior-for-unused-segments" class="hash-link" aria-label="Direct link to Behavior for unused segments" title="Direct link to Behavior for unused segments"></a></h4> |
| <p>Druid automatically retains any segments marked as unused. Previously, Druid permanently deleted unused segments from metadata store and deep storage after their duration to retain passed. This behavior was reverted from 0.23.0.</p> |
| <p><a href="https://github.com/apache/druid/pull/12693" target="_blank" rel="noopener noreferrer">#12693</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="default-for-druidprocessingfifo">Default for <code>druid.processing.fifo</code><a href="#default-for-druidprocessingfifo" class="hash-link" aria-label="Direct link to default-for-druidprocessingfifo" title="Direct link to default-for-druidprocessingfifo"></a></h4> |
| <p>The default for <code>druid.processing.fifo</code> is now true. This means that tasks of equal priority are treated in a FIFO manner. For most use cases, this change can improve performance on heavily loaded clusters.</p> |
| <p><a href="https://github.com/apache/druid/pull/12571" target="_blank" rel="noopener noreferrer">#12571</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="update-to-jdbc-statement-closure">Update to JDBC statement closure<a href="#update-to-jdbc-statement-closure" class="hash-link" aria-label="Direct link to Update to JDBC statement closure" title="Direct link to Update to JDBC statement closure"></a></h4> |
| <p>In previous releases, Druid automatically closed the JDBC Statement when the ResultSet was closed. Druid closed the ResultSet on EOF. Druid closed the statement on any exception. This behavior is, however, non-standard. |
| In this release, Druid's JDBC driver follows the JDBC standards more closely: |
| The ResultSet closes automatically on EOF, but does not close the Statement or PreparedStatement. Your code must close these statements, perhaps by using a try-with-resources block. |
| The PreparedStatement can now be used multiple times with different parameters. (Previously this was not true since closing the ResultSet closed the PreparedStatement.) |
| If any call to a Statement or PreparedStatement raises an error, the client code must still explicitly close the statement. According to the JDBC standards, statements are not closed automatically on errors. This allows you to obtain information about a failed statement before closing it. |
| If you have code that depended on the old behavior, you may have to change your code to add the required close statement.</p> |
| <p><a href="https://github.com/apache/druid/pull/12709" target="_blank" rel="noopener noreferrer">#12709</a></p> |
| <h2 class="anchor anchorWithStickyNavbar_LWe7" id="0230">0.23.0<a href="#0230" class="hash-link" aria-label="Direct link to 0.23.0" title="Direct link to 0.23.0"></a></h2> |
| <h3 class="anchor anchorWithStickyNavbar_LWe7" id="upgrade-notes-9">Upgrade notes<a href="#upgrade-notes-9" class="hash-link" aria-label="Direct link to Upgrade notes" title="Direct link to Upgrade notes"></a></h3> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="auto-killing-of-segments">Auto-killing of segments<a href="#auto-killing-of-segments" class="hash-link" aria-label="Direct link to Auto-killing of segments" title="Direct link to Auto-killing of segments"></a></h4> |
| <p>In 0.23.0, Auto killing of segments is now enabled by default <a href="https://github.com/apache/druid/pull/12187" target="_blank" rel="noopener noreferrer">(#12187)</a>. The new defaults should kill all unused segments older than 90 days. If users do not want this behavior on an upgrade, they should explicitly disable the behavior. This is a risky change since depending on the interval, segments will be killed immediately after being marked unused. this behavior will be reverted or changed in the next druid release. Please see <a href="https://github.com/apache/druid/pull/12693" target="_blank" rel="noopener noreferrer">(#12693)</a> for more details.</p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="other-changes">Other changes<a href="#other-changes" class="hash-link" aria-label="Direct link to Other changes" title="Direct link to Other changes"></a></h4> |
| <h1>Other changes</h1> |
| <ul> |
| <li>Kinesis ingestion requires <code>listShards</code> API access on the stream.</li> |
| <li>Kafka clients libraries have been upgraded to 3.0.0 <a href="https://github.com/apache/druid/pull/11735" target="_blank" rel="noopener noreferrer">(#11735)</a></li> |
| <li>The dynamic coordinator config, <code>percentOfSegmentsToConsiderPerMove</code> has been deprecated and will be removed in a future release of Druid. It is being replaced by a new segment picking strategy introduced in <a href="https://github.com/apache/druid/pull/11257" target="_blank" rel="noopener noreferrer">(#11257)</a>. This new strategy is currently toggled off by default, but can be toggled on if you set the dynamic coordinator config <code>useBatchedSegmentSampler</code> to true. Setting this as such, will disable the use of the deprecated <code>percentOfSegmentsToConsiderPerMove</code>. In a future release, <code>useBatchedSegmentSampler</code> will become permanently true. <a href="https://github.com/apache/druid/pull/11960" target="_blank" rel="noopener noreferrer">(#11960)</a></li> |
| </ul> |
| <h2 class="anchor anchorWithStickyNavbar_LWe7" id="0220">0.22.0<a href="#0220" class="hash-link" aria-label="Direct link to 0.22.0" title="Direct link to 0.22.0"></a></h2> |
| <h3 class="anchor anchorWithStickyNavbar_LWe7" id="upgrade-notes-10">Upgrade notes<a href="#upgrade-notes-10" class="hash-link" aria-label="Direct link to Upgrade notes" title="Direct link to Upgrade notes"></a></h3> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="dropped-support-for-apache-zookeeper-34">Dropped support for Apache ZooKeeper 3.4<a href="#dropped-support-for-apache-zookeeper-34" class="hash-link" aria-label="Direct link to Dropped support for Apache ZooKeeper 3.4" title="Direct link to Dropped support for Apache ZooKeeper 3.4"></a></h4> |
| <p>Following up to 0.21, which officially deprecated support for ZooKeeper 3.4, <a href="https://lists.apache.org/thread/xckr6nnsg9rxchkbvltkvt7hr2d0mhbo" target="_blank" rel="noopener noreferrer">which has been end-of-life for a while</a>, support for ZooKeeper 3.4 is now removed in 0.22.0. Be sure to upgrade your ZooKeeper cluster prior to upgrading your Druid cluster to 0.22.0.</p> |
| <p><a href="https://github.com/apache/druid/issues/10780" target="_blank" rel="noopener noreferrer">#10780</a> |
| <a href="https://github.com/apache/druid/pull/11073" target="_blank" rel="noopener noreferrer">#11073</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="native-batch-ingestion-segment-allocation-fix">Native batch ingestion segment allocation fix<a href="#native-batch-ingestion-segment-allocation-fix" class="hash-link" aria-label="Direct link to Native batch ingestion segment allocation fix" title="Direct link to Native batch ingestion segment allocation fix"></a></h4> |
| <p>Druid 0.22.0 includes an important bug-fix in native batch indexing where transient failures of indexing sub-tasks can result in non-contiguous partitions in the result segments, which will never become queryable due to logic which checks for the 'complete' set. This issue has been resolved in the latest version of Druid, but required a change in the protocol which batch tasks use to allocate segments, and this change can cause issues during rolling downgrades if you decide to roll back from Druid 0.22.0 to an earlier version.</p> |
| <p>To avoid task failure during a rolling-downgrade, set</p> |
| <div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#bfc7d5;background-color:#292d3e"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.indexer.task.default.context={ "useLineageBasedSegmentAllocation" : false }</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div> |
| <p>in the overlord runtime properties, and wait for all tasks which have <code>useLineageBasedSegmentAllocation</code> set to true to complete before initiating the downgrade. After these tasks have all completed the downgrade shouldn't have any further issue and the setting can be removed from the overlord configuration (recommended, as you will want this setting enabled if you are running Druid 0.22.0 or newer).</p> |
| <p><a href="https://github.com/apache/druid/pull/11189" target="_blank" rel="noopener noreferrer">#11189</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="sql-timeseries-no-longer-skip-empty-buckets-with-all-granularity">SQL timeseries no longer skip empty buckets with all granularity<a href="#sql-timeseries-no-longer-skip-empty-buckets-with-all-granularity" class="hash-link" aria-label="Direct link to SQL timeseries no longer skip empty buckets with all granularity" title="Direct link to SQL timeseries no longer skip empty buckets with all granularity"></a></h4> |
| <p>Prior to Druid 0.22, an SQL group by query which is using a single universal grouping key (e.g. only aggregators) such as <code>SELECT COUNT(*), SUM(x) FROM y WHERE z = 'someval'</code> would produce an empty result set instead of <code>[0, null]</code> that might be expected from this query matching no results. This was because underneath this would plan into a timeseries query with 'ALL' granularity, and skipEmptyBuckets set to true in the query context. This latter option caused the results of such a query to return no results, as there are no buckets with values to aggregate and so they are skipped, making an empty result set instead of a 'nil' result set. This behavior has been changed to behave in line with other SQL implementations, but the previous behavior can be obtained by explicitly setting <code>skipEmptyBuckets</code> on the query context.</p> |
| <p><a href="https://github.com/apache/druid/pull/11188" target="_blank" rel="noopener noreferrer">#11188</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="druid-reingestion-incompatible-changes">Druid reingestion incompatible changes<a href="#druid-reingestion-incompatible-changes" class="hash-link" aria-label="Direct link to Druid reingestion incompatible changes" title="Direct link to Druid reingestion incompatible changes"></a></h4> |
| <p>Batch tasks using a 'Druid' input source to reingest segment data will no longer accept the 'dimensions' and 'metrics' sections of their task spec, and now will internally use a new columns filter to specify which columns from the original segment should be retained. Additionally, timestampSpec is no longer ignored, allowing the __time column to be modified or replaced with a different column. These changes additionally fix a bug where transformed columns would be ignored and unavailable on the new segments.</p> |
| <p><a href="https://github.com/apache/druid/pull/10267" target="_blank" rel="noopener noreferrer">#10267</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="druid-web-console-no-longer-supports-ie11-and-other-older-browsers">Druid web-console no longer supports IE11 and other older browsers<a href="#druid-web-console-no-longer-supports-ie11-and-other-older-browsers" class="hash-link" aria-label="Direct link to Druid web-console no longer supports IE11 and other older browsers" title="Direct link to Druid web-console no longer supports IE11 and other older browsers"></a></h4> |
| <p>Some things might still work, but it is no longer officially supported so that newer Javascript features can be used to develop the web-console.</p> |
| <p><a href="https://github.com/apache/druid/pull/11357" target="_blank" rel="noopener noreferrer">#11357</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="changed-default-maximum-segment-loading-queue-size">Changed default maximum segment loading queue size<a href="#changed-default-maximum-segment-loading-queue-size" class="hash-link" aria-label="Direct link to Changed default maximum segment loading queue size" title="Direct link to Changed default maximum segment loading queue size"></a></h4> |
| <p>Druid coordinator <code>maxSegmentsInNodeLoadingQueue</code> dynamic configuration has been changed from unlimited (<code>0</code>) to <code>100</code>. This should make the coordinator behave in a much more relaxed manner during periods of cluster volatility, such as a rolling upgrade, but caps the total number of segments that will be loaded in any given coordinator cycle to 100 per server, which can slow down the speed at which a completely stopped cluster is started and loaded from deep storage.</p> |
| <p><a href="https://github.com/apache/druid/pull/11540" target="_blank" rel="noopener noreferrer">#11540</a></p> |
| <h2 class="anchor anchorWithStickyNavbar_LWe7" id="0210">0.21.0<a href="#0210" class="hash-link" aria-label="Direct link to 0.21.0" title="Direct link to 0.21.0"></a></h2> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="improved-http-status-codes-for-query-errors">Improved HTTP status codes for query errors<a href="#improved-http-status-codes-for-query-errors" class="hash-link" aria-label="Direct link to Improved HTTP status codes for query errors" title="Direct link to Improved HTTP status codes for query errors"></a></h4> |
| <p>Before this release, Druid returned the "internal error (500)" for most of the query errors. Now Druid returns different error codes based on their cause. The following table lists the errors and their corresponding codes that has changed:</p> |
| <table><thead><tr><th>Exception</th><th>Description</th><th>Old code</th><th>New code</th></tr></thead><tbody><tr><td>SqlParseException and ValidationException from Calcite</td><td>Query planning failed</td><td>500</td><td>400</td></tr><tr><td>QueryTimeoutException</td><td>Query execution didn't finish in timeout</td><td>500</td><td>504</td></tr><tr><td>ResourceLimitExceededException</td><td>Query asked more resources than configured threshold</td><td>500</td><td>400</td></tr><tr><td>InsufficientResourceException</td><td>Query failed to schedule because of lack of merge buffers available at the time when it was submitted</td><td>500</td><td>429, merged to QueryCapacityExceededException</td></tr><tr><td>QueryUnsupportedException</td><td>Unsupported functionality</td><td>400</td><td>501</td></tr></tbody></table> |
| <p><a href="https://github.com/apache/druid/pull/10464" target="_blank" rel="noopener noreferrer">#10464</a> |
| <a href="https://github.com/apache/druid/pull/10746" target="_blank" rel="noopener noreferrer">#10746</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="query-interrupted-metric">Query interrupted metric<a href="#query-interrupted-metric" class="hash-link" aria-label="Direct link to Query interrupted metric" title="Direct link to Query interrupted metric"></a></h4> |
| <p><code>query/interrupted/count</code> no longer counts the queries that timed out. These queries are counted by <code>query/timeout/count</code>.</p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="context-dimension-in-query-metrics">context dimension in query metrics<a href="#context-dimension-in-query-metrics" class="hash-link" aria-label="Direct link to context dimension in query metrics" title="Direct link to context dimension in query metrics"></a></h4> |
| <p><code>context</code> is now a default dimension emitted for all query metrics. <code>context</code> is a JSON-formatted string containing the query context for the query that the emitted metric refers to. The addition of a dimension that was not previously alters some metrics emitted by Druid. You should plan to handle this new <code>context</code> dimension in your metrics pipeline. Since the dimension is a JSON-formatted string, a common solution is to parse the dimension and either flatten it or extract the bits you want and discard the full JSON-formatted string blob.</p> |
| <p><a href="https://github.com/apache/druid/pull/10578" target="_blank" rel="noopener noreferrer">#10578</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="deprecated-support-for-apache-zookeeper-34">Deprecated support for Apache ZooKeeper 3.4<a href="#deprecated-support-for-apache-zookeeper-34" class="hash-link" aria-label="Direct link to Deprecated support for Apache ZooKeeper 3.4" title="Direct link to Deprecated support for Apache ZooKeeper 3.4"></a></h4> |
| <p>As <a href="https://mail-archives.apache.org/mod_mbox/zookeeper-user/202004.mbox/%3C41A7EC67-D8F4-4C3A-B2DB-C2741C2EECA3%40apache.org%3E" target="_blank" rel="noopener noreferrer">ZooKeeper 3.4 has been end-of-life for a while</a>, support for ZooKeeper 3.4 is deprecated in 0.21.0 and will be removed in the near future.</p> |
| <p><a href="https://github.com/apache/druid/issues/10780" target="_blank" rel="noopener noreferrer">#10780</a></p> |
| <h4 class="anchor anchorWithStickyNavbar_LWe7" id="consistent-serialization-format-and-column-naming-convention-for-the-syssegments-table">Consistent serialization format and column naming convention for the sys.segments table<a href="#consistent-serialization-format-and-column-naming-convention-for-the-syssegments-table" class="hash-link" aria-label="Direct link to Consistent serialization format and column naming convention for the sys.segments table" title="Direct link to Consistent serialization format and column naming convention for the sys.segments table"></a></h4> |
| <p>All columns in the <code>sys.segments</code> table are now serialized in the JSON format to make them consistent with other system tables. Column names now use the same "snake case" convention.</p> |
| <p><a href="https://github.com/apache/druid/pull/10481" target="_blank" rel="noopener noreferrer">#10481</a></p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/docs/latest/release-info/release-notes"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Release notes</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/docs/latest/release-info/migration-guide"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Migration guides</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#3300" class="table-of-contents__link toc-highlight">33.0.0</a><ul><li><a href="#upgrade-notes" class="table-of-contents__link toc-highlight">Upgrade notes</a></li></ul></li><li><a href="#3200" class="table-of-contents__link toc-highlight">32.0.0</a><ul><li><a href="#incompatible-changes" class="table-of-contents__link toc-highlight">Incompatible changes</a></li><li><a href="#ansi-sql-compatibility-and-query-results" class="table-of-contents__link toc-highlight">ANSI-SQL compatibility and query results</a></li><li><a href="#java-support" class="table-of-contents__link toc-highlight">Java support</a></li><li><a href="#javascript-support" class="table-of-contents__link toc-highlight">Javascript support</a></li><li><a href="#deprecations" class="table-of-contents__link toc-highlight">Deprecations</a></li><li><a href="#hadoop-based-ingestion" class="table-of-contents__link toc-highlight">Hadoop-based ingestion</a></li></ul></li><li><a href="#3100" class="table-of-contents__link toc-highlight">31.0.0</a><ul><li><a href="#upgrade-notes-1" class="table-of-contents__link toc-highlight">Upgrade notes</a></li><li><a href="#incompatible-changes-1" class="table-of-contents__link toc-highlight">Incompatible changes</a></li></ul></li><li><a href="#3000" class="table-of-contents__link toc-highlight">30.0.0</a><ul><li><a href="#upgrade-notes-2" class="table-of-contents__link toc-highlight">Upgrade notes</a></li><li><a href="#incompatible-changes-2" class="table-of-contents__link toc-highlight">Incompatible changes</a></li></ul></li><li><a href="#2901" class="table-of-contents__link toc-highlight">29.0.1</a><ul><li><a href="#incompatible-changes-3" class="table-of-contents__link toc-highlight">Incompatible changes</a></li></ul></li><li><a href="#2900" class="table-of-contents__link toc-highlight">29.0.0</a><ul><li><a href="#upgrade-notes-3" class="table-of-contents__link toc-highlight">Upgrade notes</a></li></ul></li><li><a href="#2800" class="table-of-contents__link toc-highlight">28.0.0</a><ul><li><a href="#upgrade-notes-4" class="table-of-contents__link toc-highlight">Upgrade notes</a></li><li><a href="#incompatible-changes-4" class="table-of-contents__link toc-highlight">Incompatible changes</a></li></ul></li><li><a href="#2700" class="table-of-contents__link toc-highlight">27.0.0</a><ul><li><a href="#upgrade-notes-5" class="table-of-contents__link toc-highlight">Upgrade notes</a></li><li><a href="#incompatible-changes-5" class="table-of-contents__link toc-highlight">Incompatible changes</a></li></ul></li><li><a href="#2600" class="table-of-contents__link toc-highlight">26.0.0</a><ul><li><a href="#upgrade-notes-6" class="table-of-contents__link toc-highlight">Upgrade notes</a></li><li><a href="#incompatible-changes-6" class="table-of-contents__link toc-highlight">Incompatible changes</a></li></ul></li><li><a href="#2500" class="table-of-contents__link toc-highlight">25.0.0</a><ul><li><a href="#upgrade-notes-7" class="table-of-contents__link toc-highlight">Upgrade notes</a></li><li><a href="#incompatible-changes-7" class="table-of-contents__link toc-highlight">Incompatible changes</a></li></ul></li><li><a href="#2400" class="table-of-contents__link toc-highlight">24.0.0</a><ul><li><a href="#upgrade-notes-8" class="table-of-contents__link toc-highlight">Upgrade notes</a></li></ul></li><li><a href="#0230" class="table-of-contents__link toc-highlight">0.23.0</a><ul><li><a href="#upgrade-notes-9" class="table-of-contents__link toc-highlight">Upgrade notes</a></li></ul></li><li><a href="#0220" class="table-of-contents__link toc-highlight">0.22.0</a><ul><li><a href="#upgrade-notes-10" class="table-of-contents__link toc-highlight">Upgrade notes</a></li></ul></li><li><a href="#0210" class="table-of-contents__link toc-highlight">0.21.0</a></li></ul></div></div></div></div></main></div></div></div><footer class="footer"><div class="container container-fluid"><div class="footer__bottom text--center"><div class="margin-bottom--sm"><img src="/img/favicon.png" class="footer__logo themedComponent_mlkZ themedComponent--light_NVdE"><img src="/img/favicon.png" class="footer__logo themedComponent_mlkZ themedComponent--dark_xIcU"></div><div class="footer__copyright">Copyright © 2025 Apache Software Foundation. Except where otherwise noted, licensed under CC BY-SA 4.0. Apache Druid, Druid, and the Druid logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</div></div></div></footer></div> |
| </body> |
| </html> |
