| <!doctype html> |
| <html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-current plugin-docs plugin-id-default docs-doc-id-release-info/release-notes"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="generator" content="Docusaurus v2.4.1"> |
| <title data-rh="true">Release notes | Apache® Druid</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:image" content="https://druid.apache.org/img/druid_nav.png"><meta data-rh="true" name="twitter:image" content="https://druid.apache.org/img/druid_nav.png"><meta data-rh="true" property="og:url" content="https://druid.apache.org/docs/latest/release-info/release-notes"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="current"><meta data-rh="true" name="docusaurus_tag" content="docs-default-current"><meta data-rh="true" name="docsearch:version" content="current"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-current"><meta data-rh="true" property="og:title" content="Release notes | Apache® Druid"><meta data-rh="true" name="description" content="<!--"><meta data-rh="true" property="og:description" content="<!--"><link data-rh="true" rel="icon" href="/img/favicon.png"><link data-rh="true" rel="canonical" href="https://druid.apache.org/docs/latest/release-info/release-notes"><link data-rh="true" rel="alternate" href="https://druid.apache.org/docs/latest/release-info/release-notes" hreflang="en"><link data-rh="true" rel="alternate" href="https://druid.apache.org/docs/latest/release-info/release-notes" hreflang="x-default"><link rel="preconnect" href="https://www.google-analytics.com"> |
| <link rel="preconnect" href="https://www.googletagmanager.com"> |
| <script async src="https://www.googletagmanager.com/gtag/js?id=UA-131010415-1"></script> |
| <script>function gtag(){dataLayer.push(arguments)}window.dataLayer=window.dataLayer||[],gtag("js",new Date),gtag("config","UA-131010415-1",{})</script> |
| |
| |
| |
| |
| <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"> |
| <script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><link rel="stylesheet" href="/assets/css/styles.546f39eb.css"> |
| <link rel="preload" href="/assets/js/runtime~main.91da3985.js" as="script"> |
| <link rel="preload" href="/assets/js/main.1f0e5e69.js" as="script"> |
| </head> |
| <body class="navigation-with-keyboard"> |
| <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus"> |
| <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><nav aria-label="Main" class="navbar navbar--fixed-top navbar--dark"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/druid_nav.png" alt="Apache® Druid" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/druid_nav.png" alt="Apache® Druid" class="themedImage_ToTc themedImage--dark_i4oU"></div></a></div><div class="navbar__items navbar__items--right"><a class="navbar__item navbar__link" href="/technology">Technology</a><a class="navbar__item navbar__link" href="/use-cases">Use Cases</a><a class="navbar__item navbar__link" href="/druid-powered">Powered By</a><a class="navbar__item navbar__link" href="/docs/latest/design/">Docs</a><a class="navbar__item navbar__link" href="/community/">Community</a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Apache®</a><ul class="dropdown__menu"><li><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Foundation<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://apachecon.com/?ref=druid.apache.org" target="_blank" rel="noopener noreferrer" class="dropdown__link">Events<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/licenses/" target="_blank" rel="noopener noreferrer" class="dropdown__link">License<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Thanks<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/security/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Security<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Sponsorship<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a class="navbar__item navbar__link" href="/downloads/">Download</a><div class="searchBox_ZlJk"><div class="navbar__search"><span aria-label="expand searchbar" role="button" class="search-icon" tabindex="0"></span><input type="search" id="search_input_react" placeholder="Loading..." aria-label="Search" class="navbar__search-input search-bar" disabled=""></div></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/latest/design/">Getting started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/latest/tutorials/tutorial-msq-extern">Tutorials</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/latest/design/architecture">Design</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/latest/ingestion/">Ingestion</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/latest/data-management/">Data management</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/latest/querying/sql">Querying</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/latest/api-reference/">API reference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/latest/configuration/">Configuration</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/latest/operations/web-console">Operations</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/latest/development/overview">Development</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/latest/misc/papers-and-talks">Misc</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/docs/latest/release-info/release-notes">Release info</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/docs/latest/release-info/release-notes">Release notes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/latest/release-info/upgrade-notes">Upgrade notes</a></li></ul></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Release info</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Release notes</span><meta itemprop="position" content="2"></li></ul></nav><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Release notes</h1></header><h2 class="anchor anchorWithStickyNavbar_LWe7" id="druid-2801">Druid 28.0.1<a href="#druid-2801" class="hash-link" aria-label="Direct link to Druid 28.0.1" title="Direct link to Druid 28.0.1"></a></h2><p>Apache Druid 28.0.1 is a patch release that fixes some issues in the Druid 28.0.0 release.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="bug-fixes">Bug fixes<a href="#bug-fixes" class="hash-link" aria-label="Direct link to Bug fixes" title="Direct link to Bug fixes"></a></h3><ul><li>Fixed a query caching issue for groupBy queries with multiple post-aggregation metrics <a href="https://github.com/apache/druid/pull/15402" target="_blank" rel="noopener noreferrer">#15402</a></li><li>Fixed an issue with compaction and reindex tasks failing during an upgrade due to the addition of the new task action <code>RetrieveSegmentsToReplaceAction</code>, which would not be available on the Overlord during the upgrade <a href="https://github.com/apache/druid/pull/15430" target="_blank" rel="noopener noreferrer">#15430</a></li><li>Fixed an issue with the <code>NullFilter</code> filter returning a domain regardless of the input column <a href="https://github.com/apache/druid/pull/15500" target="_blank" rel="noopener noreferrer">#15500</a></li><li>Improved the <code>start-druid</code> starter script <a href="https://github.com/apache/druid/pull/15405" target="_blank" rel="noopener noreferrer">#15405</a></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="druid-2800">Druid 28.0.0<a href="#druid-2800" class="hash-link" aria-label="Direct link to Druid 28.0.0" title="Direct link to Druid 28.0.0"></a></h2><p>Apache Druid 28.0.0 contains over 420 new features, bug fixes, performance enhancements, documentation improvements, and additional test coverage from 57 contributors.</p><p>See the <a href="https://github.com/apache/druid/issues?q=is%3Aclosed+milestone%3A28.0+sort%3Aupdated-desc+" target="_blank" rel="noopener noreferrer">complete set of changes</a> for additional details, including bug fixes.</p><p>Review the <a href="#upgrade-notes">upgrade notes</a> and <a href="#incompatible-changes">incompatible changes</a> before you upgrade to Druid 28.0.0.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="important-features-changes-and-deprecations">Important features, changes, and deprecations<a href="#important-features-changes-and-deprecations" class="hash-link" aria-label="Direct link to Important features, changes, and deprecations" title="Direct link to Important features, changes, and deprecations"></a></h2><p>In Druid 28.0.0, we have made substantial improvements to querying to make the system more ANSI SQL compatible. This includes changes in handling NULL and boolean values as well as boolean logic. At the same time, the Apache Calcite library has been upgraded to the latest version. While we have documented known query behavior changes, please read the <a href="#upgrade-notes">upgrade notes</a> section carefully. Test your application before rolling out to broad production scenarios while closely monitoring the query status.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sql-compatibility">SQL compatibility<a href="#sql-compatibility" class="hash-link" aria-label="Direct link to SQL compatibility" title="Direct link to SQL compatibility"></a></h3><p>Druid continues to make SQL query execution more consistent with how standard SQL behaves. However, there are feature flags available to restore the old behavior if needed.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="three-valued-logic">Three-valued logic<a href="#three-valued-logic" class="hash-link" aria-label="Direct link to Three-valued logic" title="Direct link to Three-valued logic"></a></h4><p>Druid native filters now observe SQL <a href="https://en.wikipedia.org/wiki/Three-valued_logic#SQL" target="_blank" rel="noopener noreferrer">three-valued logic</a> (<code>true</code>, <code>false</code>, or <code>unknown</code>) instead of Druid's classic two-state logic by default, when the following default settings apply:</p><ul><li><code>druid.generic.useThreeValueLogicForNativeFilters = true</code></li><li><code>druid.expressions.useStrictBooleans = true</code></li><li><code>druid.generic.useDefaultValueForNull = false</code></li></ul><p><a href="https://github.com/apache/druid/pull/15058" target="_blank" rel="noopener noreferrer">#15058</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="strict-booleans">Strict booleans<a href="#strict-booleans" class="hash-link" aria-label="Direct link to Strict booleans" title="Direct link to Strict booleans"></a></h4><p><code>druid.expressions.useStrictBooleans</code> is now enabled by default. |
| Druid now handles booleans strictly using <code>1</code> (true) or <code>0</code> (false). |
| Previously, true and false could be represented either as <code>true</code> and <code>false</code> as well as <code>1</code> and <code>0</code>, respectively. |
| In addition, Druid now returns a null value for Boolean comparisons like <code>True && NULL</code>.</p><p>If you don't explicitly configure this property in <code>runtime.properties</code>, clusters now use LONG types for any ingested boolean values and in the output of boolean functions for transformations and query time operations. |
| For more information, see <a href="#sql-compatibility-1">SQL compatibility in the upgrade notes</a>.</p><p><a href="https://github.com/apache/druid/pull/14734" target="_blank" rel="noopener noreferrer">#14734</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="null-handling">NULL handling<a href="#null-handling" class="hash-link" aria-label="Direct link to NULL handling" title="Direct link to NULL handling"></a></h4><p><code>druid.generic.useDefaultValueForNull</code> is now disabled by default. |
| Druid now differentiates between empty records and null records. |
| Previously, Druid might treat empty records as empty or null. |
| For more information, see <a href="#sql-compatibility-1">SQL compatibility in the upgrade notes</a>.</p><p><a href="https://github.com/apache/druid/pull/14792" target="_blank" rel="noopener noreferrer">#14792</a></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sql-planner-improvements">SQL planner improvements<a href="#sql-planner-improvements" class="hash-link" aria-label="Direct link to SQL planner improvements" title="Direct link to SQL planner improvements"></a></h3><p>Druid uses Apache Calcite for SQL planning and optimization. Starting in Druid 28.0.0, the Calcite version has been upgraded from 1.21 to 1.35. This upgrade brings in many bug fixes in SQL planning from Calcite.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="dynamic-parameters">Dynamic parameters<a href="#dynamic-parameters" class="hash-link" aria-label="Direct link to Dynamic parameters" title="Direct link to Dynamic parameters"></a></h4><p>As part of the Calcite upgrade, the behavior of type inference for dynamic parameters has changed. To avoid any type interference issues, explicitly <code>CAST</code> all dynamic parameters as a specific data type in SQL queries. For example, use:</p><div class="language-sql codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sql codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token keyword" style="font-style:italic">SELECT</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token plain"> </span><span class="token operator" style="color:rgb(137, 221, 255)">*</span><span class="token plain"> CAST </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">? </span><span class="token keyword" style="font-style:italic">as</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">DOUBLE</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token operator" style="color:rgb(137, 221, 255)">/</span><span class="token number" style="color:rgb(247, 140, 108)">2</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">as</span><span class="token plain"> tmp</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Do not use:</p><div class="language-sql codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sql codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token keyword" style="font-style:italic">SELECT</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token plain"> </span><span class="token operator" style="color:rgb(137, 221, 255)">*</span><span class="token plain"> ?</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token operator" style="color:rgb(137, 221, 255)">/</span><span class="token number" style="color:rgb(247, 140, 108)">2</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">as</span><span class="token plain"> tmp</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="async-query-and-query-from-deep-storage">Async query and query from deep storage<a href="#async-query-and-query-from-deep-storage" class="hash-link" aria-label="Direct link to Async query and query from deep storage" title="Direct link to Async query and query from deep storage"></a></h3><p><a href="https://druid.apache.org/docs/latest/querying/query-deep-storage/" target="_blank" rel="noopener noreferrer">Query from deep storage</a> is no longer an experimental feature. When you query from deep storage, more data is available for queries without having to scale your Historical services to accommodate more data. To benefit from the space saving that query from deep storage offers, configure your load rules to unload data from your Historical services.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="support-for-multiple-result-formats">Support for multiple result formats<a href="#support-for-multiple-result-formats" class="hash-link" aria-label="Direct link to Support for multiple result formats" title="Direct link to Support for multiple result formats"></a></h4><p>Query from deep storage now supports multiple result formats. |
| Previously, the <code>/druid/v2/sql/statements/</code> endpoint only supported results in the <code>object</code> format. Now, results can be written in any format specified in the <code>resultFormat</code> parameter. |
| For more information on result parameters supported by the Druid SQL API, see <a href="https://druid.apache.org/docs/latest/api-reference/sql-api#responses" target="_blank" rel="noopener noreferrer">Responses</a>.</p><p><a href="https://github.com/apache/druid/pull/14571" target="_blank" rel="noopener noreferrer">#14571</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="broadened-access-for-queries-from-deep-storage">Broadened access for queries from deep storage<a href="#broadened-access-for-queries-from-deep-storage" class="hash-link" aria-label="Direct link to Broadened access for queries from deep storage" title="Direct link to Broadened access for queries from deep storage"></a></h4><p>Users with the <code>STATE</code> permission can interact with status APIs for queries from deep storage. Previously, only the user who submitted the query could use those APIs. This enables the web console to monitor the running status of the queries. Users with the <code>STATE</code> permission can access the query results.</p><p><a href="https://github.com/apache/druid/pull/14944" target="_blank" rel="noopener noreferrer">#14944</a></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="msq-queries-for-realtime-tasks">MSQ queries for realtime tasks<a href="#msq-queries-for-realtime-tasks" class="hash-link" aria-label="Direct link to MSQ queries for realtime tasks" title="Direct link to MSQ queries for realtime tasks"></a></h3><p>The MSQ task engine can now include real time segments in query results. To do this, use the <code>includeSegmentSource</code> context parameter and set it to <code>REALTIME</code>.</p><p><a href="https://github.com/apache/druid/pull/15024" target="_blank" rel="noopener noreferrer">#15024</a></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="msq-support-for-union-all-queries">MSQ support for UNION ALL queries<a href="#msq-support-for-union-all-queries" class="hash-link" aria-label="Direct link to MSQ support for UNION ALL queries" title="Direct link to MSQ support for UNION ALL queries"></a></h3><p>You can now use the MSQ task engine to run UNION ALL queries with <code>UnionDataSource</code>.</p><p><a href="https://github.com/apache/druid/pull/14981" target="_blank" rel="noopener noreferrer">#14981</a></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="ingest-from-multiple-kafka-topics-to-a-single-datasource">Ingest from multiple Kafka topics to a single datasource<a href="#ingest-from-multiple-kafka-topics-to-a-single-datasource" class="hash-link" aria-label="Direct link to Ingest from multiple Kafka topics to a single datasource" title="Direct link to Ingest from multiple Kafka topics to a single datasource"></a></h3><p>You can now ingest streaming data from multiple Kafka topics to a datasource using a single supervisor. |
| You configure the topics for the supervisor spec using a regex pattern as the value for <code>topicPattern</code> in the IO config. If you add new topics to Kafka that match the regex, Druid automatically starts ingesting from those new topics.</p><p>If you enable multi-topic ingestion for a datasource, downgrading will cause the Supervisor to fail. |
| For more information, see <a href="#stop-supervisors-that-ingest-from-multiple-kafka-topics-before-downgrading">Stop supervisors that ingest from multiple Kafka topics before downgrading</a>.</p><p><a href="https://github.com/apache/druid/pull/14424" target="_blank" rel="noopener noreferrer">#14424</a> |
| <a href="https://github.com/apache/druid/pull/14865" target="_blank" rel="noopener noreferrer">#14865</a></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sql-unnest-and-ingestion-flattening">SQL UNNEST and ingestion flattening<a href="#sql-unnest-and-ingestion-flattening" class="hash-link" aria-label="Direct link to SQL UNNEST and ingestion flattening" title="Direct link to SQL UNNEST and ingestion flattening"></a></h3><p>The UNNEST function is no longer experimental. |
| Druid now supports UNNEST in SQL-based batch ingestion and query from deep storage, so you can flatten arrays easily. For more information, see <a href="https://druid.apache.org/docs/latest/querying/sql/#unnest" target="_blank" rel="noopener noreferrer">UNNEST</a> and <a href="https://druid.apache.org/docs/latest/tutorials/tutorial-unnest-arrays/" target="_blank" rel="noopener noreferrer">Unnest arrays within a column</a>.</p><p>You no longer need to include the context parameter <code>enableUnnest: true</code> to use UNNEST.</p><p><a href="https://github.com/apache/druid/pull/14886" target="_blank" rel="noopener noreferrer">#14886</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="recommended-syntax-for-sql-unnest">Recommended syntax for SQL UNNEST<a href="#recommended-syntax-for-sql-unnest" class="hash-link" aria-label="Direct link to Recommended syntax for SQL UNNEST" title="Direct link to Recommended syntax for SQL UNNEST"></a></h4><p>The recommended syntax for SQL UNNEST has changed. We recommend using CROSS JOIN instead of commas for most queries to prevent issues with precedence. For example, use:</p><div class="language-sql codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sql codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token keyword" style="font-style:italic">SELECT</span><span class="token plain"> column_alias_name1 </span><span class="token keyword" style="font-style:italic">FROM</span><span class="token plain"> datasource </span><span class="token keyword" style="font-style:italic">CROSS</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">JOIN</span><span class="token plain"> UNNEST</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">source_expression1</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">AS</span><span class="token plain"> table_alias_name1</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">column_alias_name1</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">CROSS</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">JOIN</span><span class="token plain"> UNNEST</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">source_expression2</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">AS</span><span class="token plain"> table_alias_name2</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">column_alias_name2</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Do not use:</p><div class="language-sql codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sql codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token keyword" style="font-style:italic">SELECT</span><span class="token plain"> column_alias_name </span><span class="token keyword" style="font-style:italic">FROM</span><span class="token plain"> datasource</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> UNNEST</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">source_expression1</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">AS</span><span class="token plain"> table_alias_name1</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">column_alias_name1</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> UNNEST</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">source_expression2</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">AS</span><span class="token plain"> table_alias_name2</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">column_alias_name2</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="window-functions-experimental">Window functions (experimental)<a href="#window-functions-experimental" class="hash-link" aria-label="Direct link to Window functions (experimental)" title="Direct link to Window functions (experimental)"></a></h3><p>You can use <a href="https://druid.apache.org/docs/latest/querying/sql-window-functions" target="_blank" rel="noopener noreferrer">window functions</a> in Apache Druid to produce values based upon the relationship of one row within a window of rows to the other rows within the same window. A window is a group of related rows within a result set. For example, rows with the same value for a specific dimension.</p><p>Enable window functions in your query with the <code>enableWindowing: true</code> context parameter.</p><p><a href="https://github.com/apache/druid/pull/15184" target="_blank" rel="noopener noreferrer">#15184</a></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="concurrent-append-and-replace-experimental">Concurrent append and replace (experimental)<a href="#concurrent-append-and-replace-experimental" class="hash-link" aria-label="Direct link to Concurrent append and replace (experimental)" title="Direct link to Concurrent append and replace (experimental)"></a></h3><p>Druid 28.0.0 adds experimental support for concurrent append and replace. |
| This feature allows you to safely replace the existing data in an interval of a datasource while new data is being appended to that interval. One of the most common applications of this is appending new data to an interval while compaction of that interval is already in progress. |
| For more information, see <a href="https://druid.apache.org/docs/latest/data-management/automatic-compaction#concurrent-append-and-replace" target="_blank" rel="noopener noreferrer">Concurrent append and replace</a>.</p><p>Segment locking will be deprecated and removed in favor of concurrent append and replace that is much simpler in design. With concurrent append and replace, Druid doesn't lock compaction jobs out because of active realtime ingestion.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="task-locks-for-append-and-replace-batch-ingestion-jobs">Task locks for append and replace batch ingestion jobs<a href="#task-locks-for-append-and-replace-batch-ingestion-jobs" class="hash-link" aria-label="Direct link to Task locks for append and replace batch ingestion jobs" title="Direct link to Task locks for append and replace batch ingestion jobs"></a></h4><p>Append batch ingestion jobs can now share locks. This allows you to run multiple append batch ingestion jobs against the same time interval. Replace batch ingestion jobs still require an exclusive lock. This means you can run multiple append batch ingestion jobs and one replace batch ingestion job for a given interval.</p><p><a href="https://github.com/apache/druid/pull/14407" target="_blank" rel="noopener noreferrer">#14407</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="streaming-ingestion-with-concurrent-replace">Streaming ingestion with concurrent replace<a href="#streaming-ingestion-with-concurrent-replace" class="hash-link" aria-label="Direct link to Streaming ingestion with concurrent replace" title="Direct link to Streaming ingestion with concurrent replace"></a></h4><p>Streaming jobs reading from Kafka and Kinesis with <code>APPEND</code> locks can now ingest concurrently with compaction running with <code>REPLACE</code> locks. The segment granularity of the streaming job must be equal to or finer than that of the concurrent replace job.</p><p><a href="https://github.com/apache/druid/pull/15039" target="_blank" rel="noopener noreferrer">#15039</a></p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="functional-area-and-related-changes">Functional area and related changes<a href="#functional-area-and-related-changes" class="hash-link" aria-label="Direct link to Functional area and related changes" title="Direct link to Functional area and related changes"></a></h2><p>This section contains detailed release notes separated by areas.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="web-console">Web console<a href="#web-console" class="hash-link" aria-label="Direct link to Web console" title="Direct link to Web console"></a></h3><h4 class="anchor anchorWithStickyNavbar_LWe7" id="added-ui-support-for-segment-loading-query-context-parameter">Added UI support for segment loading query context parameter<a href="#added-ui-support-for-segment-loading-query-context-parameter" class="hash-link" aria-label="Direct link to Added UI support for segment loading query context parameter" title="Direct link to Added UI support for segment loading query context parameter"></a></h4><p>The web console supports the <code>waitUntilSegmentsLoad</code> query context parameter.</p><p><img loading="lazy" alt="UI for waitUntilSegmentsLoad context parameter" src="/assets/images/image-401c478b5ee2980fb62c666ecb5d3cde.png" width="3010" height="2034" class="img_ev3q"></p><p><a href="https://github.com/apache/druid/pull/15110" target="_blank" rel="noopener noreferrer">#15110</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="added-concurrent-append-and-replace-switches">Added concurrent append and replace switches<a href="#added-concurrent-append-and-replace-switches" class="hash-link" aria-label="Direct link to Added concurrent append and replace switches" title="Direct link to Added concurrent append and replace switches"></a></h4><p>The web console includes concurrent append and replace switches.</p><p>The following screenshot shows the concurrent append and replace switches in the classic batch ingestion wizard: |
| <img loading="lazy" alt="Classic batch ingestion wizard" src="/assets/images/image-1-de81e1658a254b79c8a0f13609c6e41f.png" width="3008" height="2040" class="img_ev3q"></p><p>The following screenshot shows the concurrent append and replace switches in the compaction configuration UI: |
| <img loading="lazy" alt="Compaction configuration UI" src="/assets/images/image-2-ad1d55fc05f11f9b43643c38b28ea73a.png" width="3008" height="2040" class="img_ev3q"></p><p><a href="https://github.com/apache/druid/pull/15114" target="_blank" rel="noopener noreferrer">#15114</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="added-ui-support-for-ingesting-from-multiple-kafka-topics-to-a-single-datasource">Added UI support for ingesting from multiple Kafka topics to a single datasource<a href="#added-ui-support-for-ingesting-from-multiple-kafka-topics-to-a-single-datasource" class="hash-link" aria-label="Direct link to Added UI support for ingesting from multiple Kafka topics to a single datasource" title="Direct link to Added UI support for ingesting from multiple Kafka topics to a single datasource"></a></h4><p>The web console supports ingesting streaming data from multiple Kafka topics to a datasource using a single supervisor.</p><p><img loading="lazy" alt="UI for Kafka multi-topic ingestion" src="/assets/images/image-3-b0c15b3a42e3822eeb5b9f9d3f071a01.png" width="614" height="952" class="img_ev3q"></p><p><a href="https://github.com/apache/druid/pull/14833" target="_blank" rel="noopener noreferrer">#14833</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="other-web-console-improvements">Other web console improvements<a href="#other-web-console-improvements" class="hash-link" aria-label="Direct link to Other web console improvements" title="Direct link to Other web console improvements"></a></h4><ul><li>You can now copy query results from the web console directly to the clipboard <a href="https://github.com/apache/druid/pull/14889" target="_blank" rel="noopener noreferrer">#14889</a></li><li>The web console now shows the execution dialog for <code>query_controller</code> tasks in the task view instead of the generic raw task details dialog. You can still access the raw task details from the ellipsis (...) menu <a href="https://github.com/apache/druid/pull/14930" target="_blank" rel="noopener noreferrer">#14930)</a></li><li>You can now select a horizontal range in the web console time chart to modify the current WHERE clause <a href="https://github.com/apache/druid/pull/14929" target="_blank" rel="noopener noreferrer">#14929</a></li><li>You can now set dynamic query parameters in the web console <a href="https://github.com/apache/druid/pull/14921" target="_blank" rel="noopener noreferrer">#14921</a></li><li>You can now edit the Coordinator dynamic configuration in the web console <a href="https://github.com/apache/druid/pull/14791" target="_blank" rel="noopener noreferrer">#14791</a></li><li>You can now prettify SQL queries and use flatten with a Kafka input format <a href="https://github.com/apache/druid/pull/14906" target="_blank" rel="noopener noreferrer">#14906</a></li><li>A warning now appears when a CSV or TSV sample contains newlines that Druid does not accept <a href="https://github.com/apache/druid/pull/14783" target="_blank" rel="noopener noreferrer">#14783</a></li><li>You can now select a format when downloading data <a href="https://github.com/apache/druid/pull/14794" target="_blank" rel="noopener noreferrer">#14794</a></li><li>Improved the clarity of cluster default rules in the retention dialog <a href="https://github.com/apache/druid/pull/14793" target="_blank" rel="noopener noreferrer">#14793</a></li><li>The web console now detects inline queries in the query text and lets you run them individually <a href="https://github.com/apache/druid/pull/14801" target="_blank" rel="noopener noreferrer">#14810</a></li><li>You can now reset specific partition offsets for a supervisor <a href="https://github.com/apache/druid/pull/14863" target="_blank" rel="noopener noreferrer">#14863</a></li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="ingestion">Ingestion<a href="#ingestion" class="hash-link" aria-label="Direct link to Ingestion" title="Direct link to Ingestion"></a></h3><h4 class="anchor anchorWithStickyNavbar_LWe7" id="json-and-auto-column-indexer">JSON and auto column indexer<a href="#json-and-auto-column-indexer" class="hash-link" aria-label="Direct link to JSON and auto column indexer" title="Direct link to JSON and auto column indexer"></a></h4><p>The <code>json</code> column type is now equivalent to using <code>auto</code> in JSON-based batch ingestion dimension specs. Upgrade your ingestion specs to <code>json</code> to take advantage of the features and functionality of <code>auto</code>, including the following:</p><ul><li>Type specializations including ARRAY typed columns</li><li>Better support for nested arrays of strings, longs, and doubles</li><li>Smarter index utilization</li></ul><p><code>json</code> type columns created with Druid 28.0.0 are not backwards compatible with Druid versions older than 26.0.0. |
| If you upgrade from one of these versions, you can continue to write nested columns in a backwards compatible format (version 4).</p><p>For more information, see <a href="#nested-column-format">Nested column format in the upgrade notes</a>.</p><p><a href="https://github.com/apache/druid/pull/14955" target="_blank" rel="noopener noreferrer">#14955</a> |
| <a href="https://github.com/apache/druid/pull/14456" target="_blank" rel="noopener noreferrer">#14456</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="ingestion-status">Ingestion status<a href="#ingestion-status" class="hash-link" aria-label="Direct link to Ingestion status" title="Direct link to Ingestion status"></a></h4><p>Ingestion reports now include a <code>segmentLoadStatus</code> object that provides information related to the ingestion, such as duration and total segments.</p><p><a href="https://github.com/apache/druid/pull/14322" target="_blank" rel="noopener noreferrer">#14322</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="sql-based-ingestion">SQL-based ingestion<a href="#sql-based-ingestion" class="hash-link" aria-label="Direct link to SQL-based ingestion" title="Direct link to SQL-based ingestion"></a></h4><h5 class="anchor anchorWithStickyNavbar_LWe7" id="ability-to-ingest-array-types">Ability to ingest ARRAY types<a href="#ability-to-ingest-array-types" class="hash-link" aria-label="Direct link to Ability to ingest ARRAY types" title="Direct link to Ability to ingest ARRAY types"></a></h5><p>SQL-based ingestion now supports storing ARRAY typed values in <a href="https://druid.apache.org/docs/latest/querying/arrays" target="_blank" rel="noopener noreferrer">ARRAY typed columns</a> as well as storing both VARCHAR and numeric typed arrays. |
| Previously, the MSQ task engine stored ARRAY typed values as <a href="https://druid.apache.org/docs/latest/querying/multi-value-dimensions" target="_blank" rel="noopener noreferrer">multi-value dimensions</a> instead of ARRAY typed columns.</p><p>The MSQ task engine now includes the <code>arrayIngestMode</code> query context parameter, which controls how |
| <code>ARRAY</code> types are stored in Druid segments. |
| Set the <code>arrayIngestMode</code> query context parameter to <code>array</code> to ingest ARRAY types.</p><p>In Druid 28.0.0, the default mode for <code>arrayIngestMode</code> is <code>mvd</code> for backwards compatibility, which only supports VARCHAR typed arrays and stores them as multi-value dimensions. This default is subject to change in future releases.</p><p>For information on how to migrate to the new behavior, see the <a href="#ingestion-options-for-array-typed-columns">Ingestion options for ARRAY typed columns in the upgrade notes</a>. |
| For information on inserting, filtering, and grouping behavior for <code>ARRAY</code> typed columns, see <a href="https://druid.apache.org/docs/latest/querying/arrays" target="_blank" rel="noopener noreferrer">Array columns</a>.</p><p><a href="https://github.com/apache/druid/pull/15093" target="_blank" rel="noopener noreferrer">#15093</a></p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="numeric-array-type-support">Numeric array type support<a href="#numeric-array-type-support" class="hash-link" aria-label="Direct link to Numeric array type support" title="Direct link to Numeric array type support"></a></h5><p>Row-based frames and, by extension, the MSQ task engine now support numeric array types. This means that all queries consuming or producing arrays work with the MSQ task engine. Numeric arrays can also be ingested using SQL-based ingestion with MSQ. For example, queries like <code>SELECT [1, 2]</code> are valid now since they consume a numeric array instead of failing with an unsupported column type exception.</p><p><a href="https://github.com/apache/druid/pull/14900" target="_blank" rel="noopener noreferrer">#14900</a></p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="azure-blob-storage-support">Azure Blob Storage support<a href="#azure-blob-storage-support" class="hash-link" aria-label="Direct link to Azure Blob Storage support" title="Direct link to Azure Blob Storage support"></a></h5><p>Added support for Microsoft Azure Blob Storage. |
| You can now use fault tolerance and durable storage with Microsoft Azure Blob Storage. |
| For more information, see <a href="https://druid.apache.org/docs/latest/multi-stage-query/reference#durable-storage" target="_blank" rel="noopener noreferrer">Durable storage</a>.</p><p><a href="https://github.com/apache/druid/pull/14660" target="_blank" rel="noopener noreferrer">#14660</a></p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="other-sql-based-ingestion-improvements">Other SQL-based ingestion improvements<a href="#other-sql-based-ingestion-improvements" class="hash-link" aria-label="Direct link to Other SQL-based ingestion improvements" title="Direct link to Other SQL-based ingestion improvements"></a></h5><ul><li>Added a new <code>rowsPerPage</code> context parameter for the MSQ task engine. |
| Use <code>rowsPerPage</code> to limit the number of rows per page. For more information on context parameters for the MSQ task engine, see <a href="https://druid.apache.org/docs/latest/multi-stage-query/reference#context-parameters" target="_blank" rel="noopener noreferrer">Context parameters</a> <a href="https://github.com/apache/druid/pull/14994" target="_blank" rel="noopener noreferrer">#14994</a></li><li>Druid now ignores <code>ServiceClosedException</code> on <code>postCounters</code> while the controller is offline <a href="https://github.com/apache/druid/pull/14707" target="_blank" rel="noopener noreferrer">#14707</a></li><li>Improved error messages related to OVERWRITE keyword <a href="https://github.com/apache/druid/pull/14870" target="_blank" rel="noopener noreferrer">#14870</a></li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="streaming-ingestion">Streaming ingestion<a href="#streaming-ingestion" class="hash-link" aria-label="Direct link to Streaming ingestion" title="Direct link to Streaming ingestion"></a></h4><h5 class="anchor anchorWithStickyNavbar_LWe7" id="ability-to-reset-offsets-for-a-supervisor">Ability to reset offsets for a supervisor<a href="#ability-to-reset-offsets-for-a-supervisor" class="hash-link" aria-label="Direct link to Ability to reset offsets for a supervisor" title="Direct link to Ability to reset offsets for a supervisor"></a></h5><p>Added a new API endpoint <code>/druid/indexer/v1/supervisor/:supervisorId/resetOffsets</code> to reset specific partition offsets for a supervisor without resetting the entire set. |
| This endpoint clears only the specified offsets in Kafka or sequence numbers in Kinesis, prompting the supervisor to resume data reading.</p><p><a href="https://github.com/apache/druid/pull/14772" target="_blank" rel="noopener noreferrer">#14772</a></p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="other-streaming-ingestion-improvements">Other streaming ingestion improvements<a href="#other-streaming-ingestion-improvements" class="hash-link" aria-label="Direct link to Other streaming ingestion improvements" title="Direct link to Other streaming ingestion improvements"></a></h5><ul><li>Added <code>PropertyNamingStrategies</code> from Jackson to fix Hadoop ingestion and make it compatible with newer Jackson <a href="https://github.com/apache/druid/pull/14671" target="_blank" rel="noopener noreferrer">#14671</a></li><li>Added pod name to the <code>TaskLocation</code> object for Kubernetes task scheduling to make debugging easier <a href="https://github.com/apache/druid/pull/14758" target="_blank" rel="noopener noreferrer">#14758</a></li><li>Added lifecycle hooks to <code>KubernetesTaskRunner</code> <a href="https://github.com/apache/druid/pull/14790" target="_blank" rel="noopener noreferrer">#14790</a></li><li>Added new method for <code>SqlStatementResource</code> and <code>SqlTaskResource</code> to set request attribute <a href="https://github.com/apache/druid/pull/14878" target="_blank" rel="noopener noreferrer">#14878</a></li><li>Added a sampling factor for <code>DeterminePartitionsJob</code> <a href="https://github.com/apache/druid/pull/13840" target="_blank" rel="noopener noreferrer">#13840</a></li><li>Added <code>usedClusterCapacity</code> to the <code>GET</code> <code>/totalWorkerCapacity</code> response. Use this API to get the total ingestion capacity on the overlord <a href="https://github.com/apache/druid/pull/14888" target="_blank" rel="noopener noreferrer">#14888</a></li><li>Improved Kubernetes task runner performance <a href="https://github.com/apache/druid/pull/14649" target="_blank" rel="noopener noreferrer">#14649</a></li><li>Improved handling of long data source names. Previously, the Kubernetes task runner would throw an error if the name of a data source was too long <a href="https://github.com/apache/druid/pull/14620" target="_blank" rel="noopener noreferrer">#14620</a></li><li>Improved the streaming ingestion completion timeout error message <a href="https://github.com/apache/druid/pull/14636" target="_blank" rel="noopener noreferrer">#14636</a></li><li>Druid now retries fetching S3 task logs on transient S3 errors <a href="https://github.com/apache/druid/pull/14714" target="_blank" rel="noopener noreferrer">#14714</a></li><li>Druid now reports <code>task/pending/time</code> metrics for Kubernetes-based ingestion <a href="https://github.com/apache/druid/pull/14698" target="_blank" rel="noopener noreferrer">#14698</a></li><li>Druid now reports <code>k8s/peon/startup/time</code> metrics for Kubernetes-based ingestion <a href="https://github.com/apache/druid/pull/14771" target="_blank" rel="noopener noreferrer">#14771</a></li><li><code>handoffConditionTimeout</code> now defaults to 15 minutes<!-- -->—<!-- -->the default change won't affect existing supervisors <a href="https://github.com/apache/druid/pull/14539" target="_blank" rel="noopener noreferrer">#14539</a></li><li>Fixed an NPE with checkpoint parsing for streaming ingestion <a href="https://github.com/apache/druid/pull/14353" target="_blank" rel="noopener noreferrer">#14353</a></li><li>Fixed an issue with Hadoop ingestion writing arrays as <code>objects.toString</code> as a result of transform expressions <a href="https://github.com/apache/druid/pull/15127" target="_blank" rel="noopener noreferrer">#15127</a></li><li>The <code>PodTemplateTaskAdapter</code> now accounts for queryable tasks <a href="https://github.com/apache/druid/pull/14789" target="_blank" rel="noopener noreferrer">#14789</a></li><li>The rolling supervisor now restarts at <code>taskDuration</code> <a href="https://github.com/apache/druid/pull/14396" target="_blank" rel="noopener noreferrer">#14396</a></li><li>S3 <code>deleteObjects</code> requests are now retried if the failure state allows retry <a href="https://github.com/apache/druid/pull/14776" target="_blank" rel="noopener noreferrer">#14776</a></li><li>You can now ingest the name of a Kafka topic to a datasource <a href="https://github.com/apache/druid/pull/14857" target="_blank" rel="noopener noreferrer">#14857</a></li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="querying">Querying<a href="#querying" class="hash-link" aria-label="Direct link to Querying" title="Direct link to Querying"></a></h3><h4 class="anchor anchorWithStickyNavbar_LWe7" id="improved-lookup-function">Improved LOOKUP function<a href="#improved-lookup-function" class="hash-link" aria-label="Direct link to Improved LOOKUP function" title="Direct link to Improved LOOKUP function"></a></h4><p>The LOOKUP function now accepts an optional constant string as a third argument. This string is used to replace missing values in results. For example, the query <code>LOOKUP(store, 'store_to_country', 'NA')</code>, returns <code>NA</code> if the <code>store_to_country</code> value is missing for a given <code>store</code>.</p><p><a href="https://github.com/apache/druid/pull/14956" target="_blank" rel="noopener noreferrer">#14956</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="avg-function">AVG function<a href="#avg-function" class="hash-link" aria-label="Direct link to AVG function" title="Direct link to AVG function"></a></h4><p>The AVG aggregation function now returns a <code>double</code> instead of a <code>long</code>.</p><p><a href="https://github.com/apache/druid/pull/15089" target="_blank" rel="noopener noreferrer">#15089</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="improvements-to-earliest-and-latest-operators">Improvements to EARLIEST and LATEST operators<a href="#improvements-to-earliest-and-latest-operators" class="hash-link" aria-label="Direct link to Improvements to EARLIEST and LATEST operators" title="Direct link to Improvements to EARLIEST and LATEST operators"></a></h4><p>Improved EARLIEST and LATEST operators as follows:</p><ul><li>EARLIEST and LATEST operators now rewrite to EARLIEST_BY and LATEST_BY during query processing to make the <code>__time</code> column reference explicit to Calcite. <a href="https://github.com/apache/druid/pull/15095" target="_blank" rel="noopener noreferrer">#15095</a></li><li>You can now use EARLIEST/EARLIEST_BY and LATEST/LATEST_BY for STRING columns without specifying the <code>maxBytesPerValue</code> parameter. |
| If you omit the <code>maxBytesPerValue</code> parameter, the aggregations default to 1024 bytes for the buffer. <a href="https://github.com/apache/druid/pull/14848" target="_blank" rel="noopener noreferrer">#14848</a></li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="functions-for-evaluating-distinctness">Functions for evaluating distinctness<a href="#functions-for-evaluating-distinctness" class="hash-link" aria-label="Direct link to Functions for evaluating distinctness" title="Direct link to Functions for evaluating distinctness"></a></h4><p>New SQL and native query functions allow you to evaluate whether two expressions are distinct or not distinct. |
| Expressions are distinct if they have different values or if one of them is NULL. |
| Expressions are not distinct if their values are the same or if both of them are NULL.</p><p>Because the functions treat NULLs as known values when used as a comparison operator, they always return true or false even if one or both expressions are NULL.</p><p>The following table shows the difference in behavior between the equals sign (=) and IS <!-- -->[NOT]<!-- --> DISTINCT FROM:</p><table><thead><tr><th>A</th><th>B</th><th>A=B</th><th>A IS NOT DISTINCT FROM B</th></tr></thead><tbody><tr><td>0</td><td>0</td><td>true</td><td>true</td></tr><tr><td>0</td><td>1</td><td>false</td><td>false</td></tr><tr><td>0</td><td>null</td><td>unknown</td><td>false</td></tr><tr><td>null</td><td>null</td><td>unknown</td><td>true</td></tr></tbody></table><p><a href="https://github.com/apache/druid/pull/14976" target="_blank" rel="noopener noreferrer">#14976</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="functions-for-evaluating-equalities">Functions for evaluating equalities<a href="#functions-for-evaluating-equalities" class="hash-link" aria-label="Direct link to Functions for evaluating equalities" title="Direct link to Functions for evaluating equalities"></a></h4><p>New SQL and native query functions allow you to evaluate whether a condition is true or false. These functions are different from <code>x == true</code> and <code>x != true</code> in that they never return null even when the variable is null.</p><table><thead><tr><th>SQL function</th><th>Native function</th></tr></thead><tbody><tr><td><code>IS_TRUE</code></td><td><code>istrue()</code></td></tr><tr><td><code>IS_FALSE</code></td><td><code>isfalse()</code></td></tr><tr><td><code>IS_NOT_TRUE</code></td><td><code>nottrue()</code></td></tr><tr><td><code>IS_NOT_FALSE</code></td><td><code>notfalse()</code></td></tr></tbody></table><p><a href="https://github.com/apache/druid/pull/14977" target="_blank" rel="noopener noreferrer">#14977</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="function-to-decode-base64-encoded-strings">Function to decode Base64-encoded strings<a href="#function-to-decode-base64-encoded-strings" class="hash-link" aria-label="Direct link to Function to decode Base64-encoded strings" title="Direct link to Function to decode Base64-encoded strings"></a></h4><p>The new SQL and native query function, <code>decode_base64_utf8</code> decodes a Base64-encoded string and returns the UTF-8-encoded string. For example, <code>decode_base64_utf8('aGVsbG8=')</code>.</p><p><a href="https://github.com/apache/druid/pull/14943" target="_blank" rel="noopener noreferrer">#14943</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="improved-subquery-guardrail">Improved subquery guardrail<a href="#improved-subquery-guardrail" class="hash-link" aria-label="Direct link to Improved subquery guardrail" title="Direct link to Improved subquery guardrail"></a></h4><p>You can now set the <code>maxSubqueryBytes</code> guardrail to one of the following:</p><ul><li><p><code>disabled</code>: Default setting. Druid doesn't apply the guardrail around the number of bytes a subquery can generate.</p></li><li><p><code>auto</code>: Druid calculates the amount of memory to use for the materialization of results as a portion of the fixed memory of the heap.<br> |
| <!-- -->In the query context, Druid uses the following formula to determine the upper limit on the number of bytes a subquery can generate:</p><div class="language-text codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">((total JVM space - memory occupied by lookups) * 0.5) / maximum queries that the system can handle concurrently</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li><li><p>INTEGER: The number of bytes to use for materializing subquery results. Set a specific value if you understand the query patterns and want to optimize memory usage.<br> |
| <!-- -->For example, set the <code>maxSubqueryBytes</code> parameter to 300000000 (<code>300 * 1000 * 1000</code>) for a 300 MB limit. |
| Set the <code>maxSubqueryBytes</code> parameter to 314572800 (<code>300 * 1024 * 1024</code>) for a 300 MiB limit.</p></li></ul><p><a href="https://github.com/apache/druid/pull/14808" target="_blank" rel="noopener noreferrer">#14808</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="other-query-improvements">Other query improvements<a href="#other-query-improvements" class="hash-link" aria-label="Direct link to Other query improvements" title="Direct link to Other query improvements"></a></h4><ul><li>Added filters to the set of filters that work with UNNEST filter rewrite and pushdown <a href="https://github.com/apache/druid/pull/14777" target="_blank" rel="noopener noreferrer">#14777</a></li><li>Enabled <a href="https://druid.apache.org/docs/latest/querying/caching#whole-query-caching" target="_blank" rel="noopener noreferrer">whole-query caching</a> on the Broker for <a href="https://druid.apache.org/docs/latest/querying/groupbyquery.html" target="_blank" rel="noopener noreferrer">groupBy v2</a> queries <a href="https://github.com/apache/druid/pull/11595" target="_blank" rel="noopener noreferrer">#11595</a></li><li>Improved performance of EARLIEST aggregator with vectorization <a href="https://github.com/apache/druid/pull/14408" target="_blank" rel="noopener noreferrer">#14408</a></li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cluster-management">Cluster management<a href="#cluster-management" class="hash-link" aria-label="Direct link to Cluster management" title="Direct link to Cluster management"></a></h3><h4 class="anchor anchorWithStickyNavbar_LWe7" id="unused-segments">Unused segments<a href="#unused-segments" class="hash-link" aria-label="Direct link to Unused segments" title="Direct link to Unused segments"></a></h4><p>Druid now stops loading and moving segments as soon as they are marked as unused. This prevents Historical processes from spending time on superfluous loads of segments that will be unloaded later. You can mark segments as unused by a drop rule, overshadowing, or by calling <a href="https://druid.apache.org/docs/latest/api-reference/data-management-api" target="_blank" rel="noopener noreferrer">the Data management API</a>.</p><p><a href="https://github.com/apache/druid/pull/14644" target="_blank" rel="noopener noreferrer">#14644</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="encrypt-data-in-transit">Encrypt data in transit<a href="#encrypt-data-in-transit" class="hash-link" aria-label="Direct link to Encrypt data in transit" title="Direct link to Encrypt data in transit"></a></h4><p>The <code>net.spy.memcached</code> client has been replaced with the AWS ElastiCache client. This change allows Druid to encrypt data in transit using TLS.</p><p>Configure it with the following properties:</p><table><thead><tr><th>Property</th><th>Description</th><th>Default</th></tr></thead><tbody><tr><td><code>druid.cache.enableTls</code></td><td>Enable TLS based connection for Memcached client. Boolean</td><td>false</td></tr><tr><td><code>druid.cache.clientMode</code></td><td>Client Mode. Static mode requires the user to specify individual cluster nodes. Dynamic mode uses <a href="https://docs.aws.amazon.com/AmazonElastiCache/latest/mem-ug/AutoDiscovery.HowAutoDiscoveryWorks.html" target="_blank" rel="noopener noreferrer">AutoDiscovery</a> feature of AWS Memcached. String. <a href="https://docs.aws.amazon.com/AmazonElastiCache/latest/mem-ug/AutoDiscovery.Manual.html" target="_blank" rel="noopener noreferrer">"static"</a> or <a href="https://docs.aws.amazon.com/AmazonElastiCache/latest/mem-ug/AutoDiscovery.Using.ModifyApp.Java.html" target="_blank" rel="noopener noreferrer">"dynamic"</a></td><td>static</td></tr><tr><td><code>druid.cache.skipTlsHostnameVerification</code></td><td>Skip TLS Hostname Verification. Boolean.</td><td>true</td></tr></tbody></table><p><a href="https://github.com/apache/druid/pull/14827/" target="_blank" rel="noopener noreferrer">#14827</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="new-metadata-in-the-druid-segments-table">New metadata in the Druid segments table<a href="#new-metadata-in-the-druid-segments-table" class="hash-link" aria-label="Direct link to New metadata in the Druid segments table" title="Direct link to New metadata in the Druid segments table"></a></h4><p>The Druid segments table now has a column called <code>used_flag_last_updated</code> (VARCHAR (255)). This column is a UTC date string corresponding to the last time that the used column was modified.</p><p>Note that this is an incompatible change to the table. For upgrade information, see <a href="#upgrade-druid-segments-table">Upgrade Druid segments table</a>.</p><p><a href="https://github.com/apache/druid/pull/12599" target="_blank" rel="noopener noreferrer">#12599</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="other-cluster-management-improvements">Other cluster management improvements<a href="#other-cluster-management-improvements" class="hash-link" aria-label="Direct link to Other cluster management improvements" title="Direct link to Other cluster management improvements"></a></h4><ul><li>You can now use multiple console appenders in Peon logging <a href="https://github.com/apache/druid/pull/14521" target="_blank" rel="noopener noreferrer">#14521</a></li><li>Thread names of the processing pool for Indexer, Peon, and Historical processes now include the query ID <a href="https://github.com/apache/druid/pull/15059" target="_blank" rel="noopener noreferrer">#15059</a></li><li>The value for <code>replicationThrottleLimit</code> used for smart segment loading has been increased from 2% to 5% of total number of used segments. The total number of replicas in the load queue at the start of a run plus the replicas assigned in a run is kept less than or equal to the throttle limit <a href="https://github.com/apache/druid/pull/14913" target="_blank" rel="noopener noreferrer">#14913</a></li><li>The value default value for <code>balancerComputeThreads</code> is now calculated based on the number of CPUs divided by 2. Previously, the value was <code>1</code>. Smart segment loading uses this computed value <a href="https://github.com/apache/druid/pull/14902" target="_blank" rel="noopener noreferrer">#14902</a></li><li>Improved <code>InvalidNullByteFault</code> errors. They now include the output column name instead of the query column name for ease of use <a href="https://github.com/apache/druid/pull/14780" target="_blank" rel="noopener noreferrer">#14780</a></li><li>Improved the exception message when <code>DruidLeaderClient</code> doesn't find leader node <a href="https://github.com/apache/druid/pull/14775" target="_blank" rel="noopener noreferrer">#14775</a></li><li>Reduced Coordinator logging under normal operation <a href="https://github.com/apache/druid/pull/14926" target="_blank" rel="noopener noreferrer">#14926</a></li><li>Heap usage is now more predictable at very minor performance cost when using nested values <a href="https://github.com/apache/druid/pull/14919" target="_blank" rel="noopener noreferrer">#14919</a></li><li><strong>Middle Manager-less ingestion</strong>: <ul><li>The <code>sys.tasks</code> metadata table and web console now show the Kubernetes pod name rather than the peon location when using Middle Manager-less ingestion <a href="https://github.com/apache/druid/pull/14959" target="_blank" rel="noopener noreferrer">#14959</a></li><li>Added support for Middle Manager-less ingestion to migrate with zero downtime to and from <code>WorkerTaskRunners</code> that use Middle Managers <a href="https://github.com/apache/druid/pull/14918" target="_blank" rel="noopener noreferrer">#14918</a></li></ul></li><li>Druid extensions cannot bind custom Coordinator duties to the duty groups <code>IndexingServiceDuties</code> and <code>MetadataStoreManagementDuties</code> anymore. These are meant to be core coordinator built-in flows and should not be affected by custom duties. Users can still define a <code>CustomCoordinatorDuty</code> with a custom duty group and period <a href="https://github.com/apache/druid/pull/14891" target="_blank" rel="noopener noreferrer">#14891</a></li><li>Druid now adjusts <code>balancerComputeThreads</code> and <code>maxSegmentsToMove</code> automatically based on usage skew between the Historical processes in a tier <a href="https://github.com/apache/druid/pull/14584" target="_blank" rel="noopener noreferrer">#14584</a></li><li>Removed the configurable property <code>druid.coordinator.compaction.skipLockedIntervals</code> because it should always be <code>true</code> <a href="https://github.com/apache/druid/pull/14807" target="_blank" rel="noopener noreferrer">#14807</a></li><li>Updated mm-less task runner lifecycle logic to better match the logic in the HTTP and ZooKeeper worker task runners <a href="https://github.com/apache/druid/pull/14895" target="_blank" rel="noopener noreferrer">#14895</a></li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="data-management">Data management<a href="#data-management" class="hash-link" aria-label="Direct link to Data management" title="Direct link to Data management"></a></h3><h4 class="anchor anchorWithStickyNavbar_LWe7" id="alert-message-for-segment-assignments">Alert message for segment assignments<a href="#alert-message-for-segment-assignments" class="hash-link" aria-label="Direct link to Alert message for segment assignments" title="Direct link to Alert message for segment assignments"></a></h4><p>Improved alert message for segment assignments when an invalid tier is specified in a load rule or when no rule applies on a segment.</p><p><a href="https://github.com/apache/druid/pull/14696" target="_blank" rel="noopener noreferrer">#14696</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="coordinator-api-for-unused-segments">Coordinator API for unused segments<a href="#coordinator-api-for-unused-segments" class="hash-link" aria-label="Direct link to Coordinator API for unused segments" title="Direct link to Coordinator API for unused segments"></a></h4><p>Added <code>includeUnused</code> as an optional parameter to the Coordinator API. |
| You can send a <code>GET</code> request to <code>/druid/coordinator/v1/metadata/datasources/{dataSourceName}/segments/{segmentId}?includeUnused=true</code> to retrieve the metadata for a specific segment as stored in the metadata store.</p><p>The API also returns unused segments if the <code>includeUnused</code> parameter is set.</p><p><a href="https://github.com/apache/druid/pull/14846/" target="_blank" rel="noopener noreferrer">#14846</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="kill-task-improvements">Kill task improvements<a href="#kill-task-improvements" class="hash-link" aria-label="Direct link to Kill task improvements" title="Direct link to Kill task improvements"></a></h4><ul><li>Added <code>killTaskSlotRatio</code> and <code>maxKillTaskSlots</code> dynamic configuration properties to allow control of task resource usage spawned by the <code>KillUnusedSegments</code> coordinator task <a href="https://github.com/apache/druid/pull/14769" target="_blank" rel="noopener noreferrer">#14769</a></li><li>The value for <code>druid.coordinator.kill.period</code> can now be greater than or equal to <code>druid.coordinator.period.indexingPeriod</code>. Previously, it had to be greater than <code>druid.coordinator.period.indexingPeriod</code>. Additionally, the leader Coordinator now keeps track of the last submitted <code>kill</code> task for a datasource to avoid submitting duplicate <code>kill</code> tasks <a href="https://github.com/apache/druid/pull/14831" target="_blank" rel="noopener noreferrer">#14831</a></li><li>Added a new config <code>druid.coordinator.kill.bufferPeriod</code> for a buffer period. This config defines the amount of time that a segment is unused before <code>KillUnusedSegment</code> can kill it. Using the default <code>PT24H</code>, if you mark a segment as unused at <code>2022-06-01T00:05:00.000Z</code>, then the segment cannot be killed until at or after <code>2022-06-02T00:05:00.000Z</code> <a href="https://github.com/apache/druid/pull/12599" target="_blank" rel="noopener noreferrer">#12599</a></li><li>You can now specify the following parameters for a <code>kill</code> task:<ul><li><code>batchSize</code>: The maximum number of segments to delete in one <code>kill</code> batch <a href="https://github.com/apache/druid/pull/14642" target="_blank" rel="noopener noreferrer">#14642</a></li><li><code>limit</code>: The maximum number of segments for a <code>kill</code> task to delete <a href="https://github.com/apache/druid/pull/14662" target="_blank" rel="noopener noreferrer">#14662</a></li></ul></li><li>You can now speed up <code>kill</code> tasks by batch deleting multiple segments stored in S3 <a href="https://github.com/apache/druid/pull/14131" target="_blank" rel="noopener noreferrer">#14131</a></li><li>Kill tasks that delete unused segments now publish a task report containing kill stats such as <code>numSegmentsKilled</code>, <code>numBatchesProcessed</code>, and <code>numSegmentsMarkedAsUnused</code> <a href="https://github.com/apache/druid/pull/15023" target="_blank" rel="noopener noreferrer">#15023</a></li><li><code>IndexerSQLMetadataStorageCoordinator</code> now uses the JDBI <code>PreparedBatch</code> instead of issuing single update statements inside a transaction to mitigate scaling challenges <a href="https://github.com/apache/druid/pull/14639" target="_blank" rel="noopener noreferrer">#14639</a></li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="metrics-and-monitoring">Metrics and monitoring<a href="#metrics-and-monitoring" class="hash-link" aria-label="Direct link to Metrics and monitoring" title="Direct link to Metrics and monitoring"></a></h3><h4 class="anchor anchorWithStickyNavbar_LWe7" id="new-ingestion-metrics">New ingestion metrics<a href="#new-ingestion-metrics" class="hash-link" aria-label="Direct link to New ingestion metrics" title="Direct link to New ingestion metrics"></a></h4><table><thead><tr><th>Metric</th><th>Description</th><th>Dimensions</th><th>Normal value</th></tr></thead><tbody><tr><td><code>ingest/input/bytes</code></td><td>Number of bytes read from input sources, after decompression but prior to parsing. This covers all data read, including data that does not end up being fully processed and ingested. For example, this includes data that ends up being rejected for being unparseable or filtered out.</td><td><code>dataSource</code>, <code>taskId</code>, <code>taskType</code>, <code>groupId</code>, <code>tags</code></td><td>Depends on the amount of data read.</td></tr></tbody></table><p><a href="https://github.com/apache/druid/pull/14582" target="_blank" rel="noopener noreferrer">#14582</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="new-query-metrics">New query metrics<a href="#new-query-metrics" class="hash-link" aria-label="Direct link to New query metrics" title="Direct link to New query metrics"></a></h4><table><thead><tr><th>Metric</th><th>Description</th><th>Dimensions</th><th>Normal value</th></tr></thead><tbody><tr><td><code>mergeBuffer/pendingRequests</code></td><td>Number of requests waiting to acquire a batch of buffers from the merge buffer pool.</td><td>This metric is exposed through the <code>QueryCountStatsMonitor</code> module for the Broker.</td><td></td></tr></tbody></table><p><a href="https://github.com/apache/druid/pull/15025" target="_blank" rel="noopener noreferrer">#15025</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="new-zookeeper-metrics">New ZooKeeper metrics<a href="#new-zookeeper-metrics" class="hash-link" aria-label="Direct link to New ZooKeeper metrics" title="Direct link to New ZooKeeper metrics"></a></h4><table><thead><tr><th>Metric</th><th>Description</th><th>Dimensions</th><th>Normal value</th></tr></thead><tbody><tr><td><code>zk/connected</code></td><td>Indicator of connection status. <code>1</code> for connected, <code>0</code> for disconnected. Emitted once per monitor period.</td><td>None</td><td>1</td></tr><tr><td><code>zk/reconnect/time</code></td><td>Amount of time, in milliseconds, that a server was disconnected from ZooKeeper before reconnecting. Emitted on reconnection. Not emitted if connection to ZooKeeper is permanently lost, because in this case, there is no reconnection.</td><td>None</td><td>Not present</td></tr></tbody></table><p><a href="https://github.com/apache/druid/pull/14333" target="_blank" rel="noopener noreferrer">#14333</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="new-subquery-metrics-for-the-broker">New subquery metrics for the Broker<a href="#new-subquery-metrics-for-the-broker" class="hash-link" aria-label="Direct link to New subquery metrics for the Broker" title="Direct link to New subquery metrics for the Broker"></a></h4><p>The new <code>SubqueryCountStatsMonitor</code> emits metrics corresponding to the subqueries and their execution.</p><table><thead><tr><th>Metric</th><th>Description</th><th>Dimensions</th><th>Normal value</th></tr></thead><tbody><tr><td><code>subquery/rowLimit/count</code></td><td>Number of subqueries whose results are materialized as rows (Java objects on heap).</td><td>This metric is only available if the <code>SubqueryCountStatsMonitor</code> module is included.</td><td></td></tr><tr><td><code>subquery/byteLimit/count</code></td><td>Number of subqueries whose results are materialized as frames (Druid's internal byte representation of rows).</td><td>This metric is only available if the <code>SubqueryCountStatsMonitor</code> module is included.</td><td></td></tr><tr><td><code>subquery/fallback/count</code></td><td>Number of subqueries which cannot be materialized as frames</td><td>This metric is only available if the <code>SubqueryCountStatsMonitor</code> module is included.</td><td></td></tr><tr><td><code>subquery/fallback/insufficientType/count</code></td><td>Number of subqueries which cannot be materialized as frames due to insufficient type information in the row signature.</td><td>This metric is only available if the <code>SubqueryCountStatsMonitor</code> module is included.</td><td></td></tr><tr><td><code>subquery/fallback/unknownReason/count</code></td><td>Number of subqueries which cannot be materialized as frames due other reasons.</td><td>This metric is only available if the <code>SubqueryCountStatsMonitor</code> module is included.</td><td></td></tr><tr><td><code>query/rowLimit/exceeded/count</code></td><td>Number of queries whose inlined subquery results exceeded the given row limit</td><td>This metric is only available if the <code>SubqueryCountStatsMonitor</code> module is included.</td><td></td></tr><tr><td><code>query/byteLimit/exceeded/count</code></td><td>Number of queries whose inlined subquery results exceeded the given byte limit</td><td>This metric is only available if the <code>SubqueryCountStatsMonitor</code> module is included.</td><td></td></tr></tbody></table><p><a href="https://github.com/apache/druid/pull/14808" target="_blank" rel="noopener noreferrer">#14808</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="new-coordinator-metrics">New Coordinator metrics<a href="#new-coordinator-metrics" class="hash-link" aria-label="Direct link to New Coordinator metrics" title="Direct link to New Coordinator metrics"></a></h4><table><thead><tr><th>Metric</th><th>Description</th><th>Dimensions</th><th>Normal value</th></tr></thead><tbody><tr><td><code>killTask/availableSlot/count</code></td><td>Number of available task slots that can be used for auto kill tasks in the auto kill run. This is the max number of task slots minus any currently running auto kill tasks.</td><td></td><td>Varies</td></tr><tr><td><code>killTask/maxSlot/count</code></td><td>Maximum number of task slots available for auto kill tasks in the auto kill run.</td><td></td><td>Varies</td></tr><tr><td><code>kill/task/count</code></td><td>Number of tasks issued in the auto kill run.</td><td></td><td>Varies</td></tr><tr><td><code>kill/pendingSegments/count</code></td><td>Number of stale pending segments deleted from the metadata store.</td><td><code>dataSource</code></td><td>Varies</td></tr></tbody></table><p><a href="https://github.com/apache/druid/pull/14782" target="_blank" rel="noopener noreferrer">#14782</a> |
| <a href="https://github.com/apache/druid/pull/14951" target="_blank" rel="noopener noreferrer">#14951</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="new-compaction-metrics">New compaction metrics<a href="#new-compaction-metrics" class="hash-link" aria-label="Direct link to New compaction metrics" title="Direct link to New compaction metrics"></a></h4><table><thead><tr><th>Metric</th><th>Description</th><th>Dimensions</th><th>Normal value</th></tr></thead><tbody><tr><td><code>compact/segmentAnalyzer/fetchAndProcessMillis</code></td><td>Time taken to fetch and process segments to infer the schema for the compaction task to run.</td><td><code>dataSource</code>, <code>taskId</code>, <code>taskType</code>, <code>groupId</code>,<code>tags</code></td><td>Varies. A high value indicates compaction tasks will speed up from explicitly setting the data schema.</td></tr></tbody></table><p><a href="https://github.com/apache/druid/pull/14752" target="_blank" rel="noopener noreferrer">#14752</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="segment-scan-metrics">Segment scan metrics<a href="#segment-scan-metrics" class="hash-link" aria-label="Direct link to Segment scan metrics" title="Direct link to Segment scan metrics"></a></h4><p>Added a new metric to figure out the usage of <code>druid.processing.numThreads</code> on the Historicals/Indexers/Peons.</p><table><thead><tr><th>Metric</th><th>Description</th><th>Dimensions</th><th>Normal value</th></tr></thead><tbody><tr><td><code>segment/scan/active</code></td><td>Number of segments currently scanned. This metric also indicates how many threads from <code>druid.processing.numThreads</code> are currently being used.</td><td></td><td>Close to <code>druid.processing.numThreads</code></td></tr></tbody></table><h4 class="anchor anchorWithStickyNavbar_LWe7" id="new-kafka-consumer-metrics">New Kafka consumer metrics<a href="#new-kafka-consumer-metrics" class="hash-link" aria-label="Direct link to New Kafka consumer metrics" title="Direct link to New Kafka consumer metrics"></a></h4><p>Added the following Kafka consumer metrics:</p><ul><li><code>kafka/consumer/bytesConsumed</code>: Equivalent to the Kafka consumer metric <code>bytes-consumed-total</code>. Only emitted for Kafka tasks.</li><li><code>kafka/consumer/recordsConsumed</code>: Equivalent to the Kafka consumer metric <code>records-consumed-total</code>. Only emitted for Kafka tasks.</li></ul><p><a href="https://github.com/apache/druid/pull/14582" target="_blank" rel="noopener noreferrer">#14582</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="serviceheartbeat-metrics">service/heartbeat metrics<a href="#serviceheartbeat-metrics" class="hash-link" aria-label="Direct link to service/heartbeat metrics" title="Direct link to service/heartbeat metrics"></a></h4><ul><li>Exposed <code>service/heartbeat</code> metric to <code>statsd-reporter</code> <a href="https://github.com/apache/druid/pull/14564" target="_blank" rel="noopener noreferrer">#14564</a></li><li>Modified the <code>service/heartbeat</code> metric to expose the <code>leader</code> dimension <a href="https://github.com/apache/druid/pull/14593" target="_blank" rel="noopener noreferrer">#14593</a></li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="tombstone-and-segment-counts">Tombstone and segment counts<a href="#tombstone-and-segment-counts" class="hash-link" aria-label="Direct link to Tombstone and segment counts" title="Direct link to Tombstone and segment counts"></a></h4><p>Added <code>ingest/tombstones/count</code> and <code>ingest/segments/count</code> metrics in MSQ to report the number of tombstones and segments after Druid finishes publishing segments.</p><p><a href="https://github.com/apache/druid/pull/14980" target="_blank" rel="noopener noreferrer">#14980</a></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="extensions">Extensions<a href="#extensions" class="hash-link" aria-label="Direct link to Extensions" title="Direct link to Extensions"></a></h3><h4 class="anchor anchorWithStickyNavbar_LWe7" id="ingestion-task-payloads-for-kubernetes">Ingestion task payloads for Kubernetes<a href="#ingestion-task-payloads-for-kubernetes" class="hash-link" aria-label="Direct link to Ingestion task payloads for Kubernetes" title="Direct link to Ingestion task payloads for Kubernetes"></a></h4><p>You can now provide compressed task payloads larger than 128 KB when you run MiddleManager-less ingestion jobs.</p><p><a href="https://github.com/apache/druid/pull/14887" target="_blank" rel="noopener noreferrer">#14887</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="prometheus-emitter">Prometheus emitter<a href="#prometheus-emitter" class="hash-link" aria-label="Direct link to Prometheus emitter" title="Direct link to Prometheus emitter"></a></h4><p>The Prometheus emitter now supports a new optional configuration parameter, <code>druid.emitter.prometheus.extraLabels</code>. |
| This addition offers the flexibility to add arbitrary extra labels to Prometheus metrics, providing more granular control in managing and identifying data across multiple Druid clusters or other dimensions. |
| For more information, see <a href="https://druid.apache.org/docs/latest/development/extensions-contrib/prometheus" target="_blank" rel="noopener noreferrer">Prometheus emitter extension</a>.</p><p><a href="https://github.com/apache/druid/pull/14728" target="_blank" rel="noopener noreferrer">#14728</a></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="documentation-improvements">Documentation improvements<a href="#documentation-improvements" class="hash-link" aria-label="Direct link to Documentation improvements" title="Direct link to Documentation improvements"></a></h3><p>We've moved Jupyter notebooks that guide you through query, ingestion, and data management with Apache Druid to the new <a href="https://github.com/implydata/learn-druid" target="_blank" rel="noopener noreferrer">Learn Druid</a> repository. |
| The repository also contains a Docker Compose file to get you up and running with a learning lab.</p><p><a href="https://github.com/apache/druid/pull/15136" target="_blank" rel="noopener noreferrer">#15136</a></p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="upgrade-notes-and-incompatible-changes">Upgrade notes and incompatible changes<a href="#upgrade-notes-and-incompatible-changes" class="hash-link" aria-label="Direct link to Upgrade notes and incompatible changes" title="Direct link to Upgrade notes and incompatible changes"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="upgrade-notes">Upgrade notes<a href="#upgrade-notes" class="hash-link" aria-label="Direct link to Upgrade notes" title="Direct link to Upgrade notes"></a></h3><h4 class="anchor anchorWithStickyNavbar_LWe7" id="upgrade-druid-segments-table">Upgrade Druid segments table<a href="#upgrade-druid-segments-table" class="hash-link" aria-label="Direct link to Upgrade Druid segments table" title="Direct link to Upgrade Druid segments table"></a></h4><p>Druid 28.0.0 adds a new column to the Druid metadata table that requires an update to the table.</p><p>If <code>druid.metadata.storage.connector.createTables</code> is set to <code>true</code> and the metadata store user has DDL privileges, the segments table gets automatically updated at startup to include the new <code>used_flag_last_updated</code> column. No additional work is needed for the upgrade.</p><p>If either of those requirements are not met, pre-upgrade steps are required. You must make these updates before you upgrade to Druid 28.0.0, or the Coordinator and Overlord processes fail.</p><p>Although you can manually alter your table to add the new <code>used_flag_last_updated</code> column, Druid also provides a CLI tool to do it.</p><p><a href="https://github.com/apache/druid/pull/12599" target="_blank" rel="noopener noreferrer">#12599</a></p><div class="theme-admonition theme-admonition-warning alert alert--danger admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 12 16"><path fill-rule="evenodd" d="M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"></path></svg></span>danger</div><div class="admonitionContent_S0QG"><p>If your metadata store doesn’t support instant ADD COLUMN semantics, like versions of MySQL prior to 8.0, and you have a very large segments table, adding a column to segments table can take a long time, block other metadata operations, and use large amounts of disk.</p></div></div><p>In the example commands below:</p><ul><li><code>lib</code> is the Druid lib directory</li><li><code>extensions</code> is the Druid extensions directory</li><li><code>base</code> corresponds to the value of <code>druid.metadata.storage.tables.base</code> in the configuration, <code>druid</code> by default.</li><li>The <code>--connectURI</code> parameter corresponds to the value of <code>druid.metadata.storage.connector.connectURI</code>.</li><li>The <code>--user</code> parameter corresponds to the value of <code>druid.metadata.storage.connector.user</code>.</li><li>The <code>--password</code> parameter corresponds to the value of <code>druid.metadata.storage.connector.password</code>.</li><li>The <code>--action</code> parameter corresponds to the update action you are executing. In this case, it is <code>add-last-used-to-segments</code></li></ul><h5 class="anchor anchorWithStickyNavbar_LWe7" id="upgrade-step-for-mysql">Upgrade step for MySQL<a href="#upgrade-step-for-mysql" class="hash-link" aria-label="Direct link to Upgrade step for MySQL" title="Direct link to Upgrade step for MySQL"></a></h5><div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token builtin class-name" style="color:rgb(255, 203, 107)">cd</span><span class="token plain"> </span><span class="token variable" style="color:rgb(191, 199, 213)">${DRUID_ROOT}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">java -classpath </span><span class="token string" style="color:rgb(195, 232, 141)">"lib/*"</span><span class="token plain"> -Dlog4j.configurationFile</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain">conf/druid/cluster/_common/log4j2.xml -Ddruid.extensions.directory</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token string" style="color:rgb(195, 232, 141)">"extensions"</span><span class="token plain"> -Ddruid.extensions.loadList</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token punctuation" style="color:rgb(199, 146, 234)">\</span><span class="token plain">"mysql-metadata-storage</span><span class="token punctuation" style="color:rgb(199, 146, 234)">\</span><span class="token plain">"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"> -Ddruid.metadata.storage.type</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain">mysql org.apache.druid.cli.Main tools metadata-update --connectURI</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token string" style="color:rgb(195, 232, 141)">"<mysql-uri>"</span><span class="token plain"> --user </span><span class="token environment constant" style="color:rgb(130, 170, 255)">USER</span><span class="token plain"> --password PASSWORD --base druid --action add-used-flag-last-updated-to-segments</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h5 class="anchor anchorWithStickyNavbar_LWe7" id="upgrade-step-for-postgresql">Upgrade step for PostgreSQL<a href="#upgrade-step-for-postgresql" class="hash-link" aria-label="Direct link to Upgrade step for PostgreSQL" title="Direct link to Upgrade step for PostgreSQL"></a></h5><div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token builtin class-name" style="color:rgb(255, 203, 107)">cd</span><span class="token plain"> </span><span class="token variable" style="color:rgb(191, 199, 213)">${DRUID_ROOT}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">java -classpath </span><span class="token string" style="color:rgb(195, 232, 141)">"lib/*"</span><span class="token plain"> -Dlog4j.configurationFile</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain">conf/druid/cluster/_common/log4j2.xml -Ddruid.extensions.directory</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token string" style="color:rgb(195, 232, 141)">"extensions"</span><span class="token plain"> -Ddruid.extensions.loadList</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token punctuation" style="color:rgb(199, 146, 234)">\</span><span class="token plain">"postgresql-metadata-storage</span><span class="token punctuation" style="color:rgb(199, 146, 234)">\</span><span class="token plain">"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"> -Ddruid.metadata.storage.type</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain">postgresql org.apache.druid.cli.Main tools metadata-update --connectURI</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token string" style="color:rgb(195, 232, 141)">"<postgresql-uri>"</span><span class="token plain"> --user </span><span class="token environment constant" style="color:rgb(130, 170, 255)">USER</span><span class="token plain"> --password PASSWORD --base druid --action add-used-flag-last-updated-to-segments</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h5 class="anchor anchorWithStickyNavbar_LWe7" id="manual-upgrade-step">Manual upgrade step<a href="#manual-upgrade-step" class="hash-link" aria-label="Direct link to Manual upgrade step" title="Direct link to Manual upgrade step"></a></h5><div class="language-SQL codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-SQL codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">ALTER TABLE druid_segments</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">ADD used_flag_last_updated varchar(255);</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="recommended-syntax-for-sql-unnest-1">Recommended syntax for SQL UNNEST<a href="#recommended-syntax-for-sql-unnest-1" class="hash-link" aria-label="Direct link to Recommended syntax for SQL UNNEST" title="Direct link to Recommended syntax for SQL UNNEST"></a></h4><p>The recommended syntax for SQL UNNEST has changed. We recommend using CROSS JOIN instead of commas for most queries to prevent issues with precedence. For example, use:</p><div class="language-sql codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sql codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token keyword" style="font-style:italic">SELECT</span><span class="token plain"> column_alias_name1 </span><span class="token keyword" style="font-style:italic">FROM</span><span class="token plain"> datasource </span><span class="token keyword" style="font-style:italic">CROSS</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">JOIN</span><span class="token plain"> UNNEST</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">source_expression1</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">AS</span><span class="token plain"> table_alias_name1</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">column_alias_name1</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">CROSS</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">JOIN</span><span class="token plain"> UNNEST</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">source_expression2</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">AS</span><span class="token plain"> table_alias_name2</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">column_alias_name2</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Do not use:</p><div class="language-sql codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sql codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token keyword" style="font-style:italic">SELECT</span><span class="token plain"> column_alias_name </span><span class="token keyword" style="font-style:italic">FROM</span><span class="token plain"> datasource</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> UNNEST</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">source_expression1</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">AS</span><span class="token plain"> table_alias_name1</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">column_alias_name1</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> UNNEST</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">source_expression2</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">AS</span><span class="token plain"> table_alias_name2</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">column_alias_name2</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="dynamic-parameters-1">Dynamic parameters<a href="#dynamic-parameters-1" class="hash-link" aria-label="Direct link to Dynamic parameters" title="Direct link to Dynamic parameters"></a></h4><p>The Apache Calcite version has been upgraded from 1.21 to 1.35. As part of the Calcite upgrade, the behavior of type inference for dynamic parameters has changed. To avoid any type interference issues, explicitly <code>CAST</code> all dynamic parameters as a specific data type in SQL queries. For example, use:</p><div class="language-sql codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sql codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token keyword" style="font-style:italic">SELECT</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token plain"> </span><span class="token operator" style="color:rgb(137, 221, 255)">*</span><span class="token plain"> CAST </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">? </span><span class="token keyword" style="font-style:italic">as</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">DOUBLE</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token operator" style="color:rgb(137, 221, 255)">/</span><span class="token number" style="color:rgb(247, 140, 108)">2</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">as</span><span class="token plain"> tmp</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Do not use:</p><div class="language-sql codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sql codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token keyword" style="font-style:italic">SELECT</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token plain"> </span><span class="token operator" style="color:rgb(137, 221, 255)">*</span><span class="token plain"> ?</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token operator" style="color:rgb(137, 221, 255)">/</span><span class="token number" style="color:rgb(247, 140, 108)">2</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">as</span><span class="token plain"> tmp</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="nested-column-format">Nested column format<a href="#nested-column-format" class="hash-link" aria-label="Direct link to Nested column format" title="Direct link to Nested column format"></a></h4><p><code>json</code> type columns created with Druid 28.0.0 are not backwards compatible with Druid versions older than 26.0.0. |
| If you are upgrading from a version prior to Druid 26.0.0 and you use <code>json</code> columns, upgrade to Druid 26.0.0 before you upgrade to Druid 28.0.0. |
| Additionally, to downgrade to a version older than Druid 26.0.0, any new segments created in Druid 28.0.0 should be re-ingested using Druid 26.0.0 or 27.0.0 prior to further downgrading.</p><p>When upgrading from a previous version, you can continue to write nested columns in a backwards compatible format (version 4).</p><p>In a classic batch ingestion job, include <code>formatVersion</code> in the <code>dimensions</code> list of the <code>dimensionsSpec</code> property. For example:</p><div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"dimensionsSpec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"dimensions"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"product"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"department"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"json"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"name"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"shipTo"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"formatVersion"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">4</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>To set the default nested column version, set the desired format version in the common runtime properties. For example:</p><div class="language-java codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-java codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.indexing.formats.nestedColumnFormatVersion=4</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="sql-compatibility-1">SQL compatibility<a href="#sql-compatibility-1" class="hash-link" aria-label="Direct link to SQL compatibility" title="Direct link to SQL compatibility"></a></h4><p>Starting with Druid 28.0.0, the default way Druid treats nulls and booleans has changed.</p><p>For nulls, Druid now differentiates between an empty string and a record with no data as well as between an empty numerical record and <code>0</code>.<br> |
| <!-- -->You can revert to the previous behavior by setting <code>druid.generic.useDefaultValueForNull</code> to <code>true</code>.</p><p>This property affects both storage and querying, and must be set on all Druid service types to be available at both ingestion time and query time. Reverting this setting to the old value restores the previous behavior without reingestion.</p><p>For booleans, Druid now strictly uses <code>1</code> (true) or <code>0</code> (false). Previously, true and false could be represented either as <code>true</code> and <code>false</code> as well as <code>1</code> and <code>0</code>, respectively. In addition, Druid now returns a null value for boolean comparisons like <code>True && NULL</code>.</p><p>You can revert to the previous behavior by setting <code>druid.expressions.useStrictBooleans</code> to <code>false</code>. |
| This property affects both storage and querying, and must be set on all Druid service types to be available at both ingestion time and query time. Reverting this setting to the old value restores the previous behavior without reingestion.</p><p>The following table illustrates some example scenarios and the impact of the changes.</p><details class="details_lb9f alert alert--info details_b_Ee" data-collapsed="true"><summary>Show the table</summary><div><div class="collapsibleContent_i85q"><table><thead><tr><th>Query</th><th>Druid 27.0.0 and earlier</th><th>Druid 28.0.0 and later</th></tr></thead><tbody><tr><td>Query empty string</td><td>Empty string (<code>''</code>) or null</td><td>Empty string (<code>''</code>)</td></tr><tr><td>Query null string</td><td>Null or empty</td><td>Null</td></tr><tr><td>COUNT(*)</td><td>All rows, including nulls</td><td>All rows, including nulls</td></tr><tr><td>COUNT(column)</td><td>All rows excluding empty strings</td><td>All rows including empty strings but excluding nulls</td></tr><tr><td>Expression 100 && 11</td><td>11</td><td>1</td></tr><tr><td>Expression 100 <!-- -->|<!-- -->|<!-- --> 11</td><td>100</td><td>1</td></tr><tr><td>Null FLOAT/DOUBLE column</td><td>0.0</td><td>Null</td></tr><tr><td>Null LONG column</td><td>0</td><td>Null</td></tr><tr><td>Null <code>__time</code> column</td><td>0, meaning 1970-01-01 00:00:00 UTC</td><td>1970-01-01 00:00:00 UTC</td></tr><tr><td>Null MVD column</td><td><code>''</code></td><td>Null</td></tr><tr><td>ARRAY</td><td>Null</td><td>Null</td></tr><tr><td>COMPLEX</td><td>none</td><td>Null</td></tr></tbody></table></div></div></details><p>Before upgrading to Druid 28.0.0, update your queries to account for the changed behavior as described in the following sections.</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="null-filters">NULL filters<a href="#null-filters" class="hash-link" aria-label="Direct link to NULL filters" title="Direct link to NULL filters"></a></h5><p>If your queries use NULL in the filter condition to match both nulls and empty strings, you should add an explicit filter clause for empty strings. For example, update <code>s IS NULL</code> to <code>s IS NULL OR s = ''</code>.</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="count-functions">COUNT functions<a href="#count-functions" class="hash-link" aria-label="Direct link to COUNT functions" title="Direct link to COUNT functions"></a></h5><p><code>COUNT(column)</code> now counts empty strings. If you want to continue excluding empty strings from the count, replace <code>COUNT(column)</code> with <code>COUNT(column) FILTER(WHERE column <> '')</code>.</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="groupby-queries">GroupBy queries<a href="#groupby-queries" class="hash-link" aria-label="Direct link to GroupBy queries" title="Direct link to GroupBy queries"></a></h5><p>GroupBy queries on columns containing null values can now have additional entries as nulls can co-exist with empty strings.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="stop-supervisors-that-ingest-from-multiple-kafka-topics-before-downgrading">Stop Supervisors that ingest from multiple Kafka topics before downgrading<a href="#stop-supervisors-that-ingest-from-multiple-kafka-topics-before-downgrading" class="hash-link" aria-label="Direct link to Stop Supervisors that ingest from multiple Kafka topics before downgrading" title="Direct link to Stop Supervisors that ingest from multiple Kafka topics before downgrading"></a></h4><p>If you have added supervisors that ingest from multiple Kafka topics in Druid 28.0.0 or later, stop those supervisors before downgrading to a version prior to Druid 28.0.0 because the supervisors will fail in versions prior to Druid 28.0.0.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="lenientaggregatormerge-deprecated"><code>lenientAggregatorMerge</code> deprecated<a href="#lenientaggregatormerge-deprecated" class="hash-link" aria-label="Direct link to lenientaggregatormerge-deprecated" title="Direct link to lenientaggregatormerge-deprecated"></a></h4><p><code>lenientAggregatorMerge</code> property in segment metadata queries has been deprecated. It will be removed in future releases. |
| Use <code>aggregatorMergeStrategy</code> instead. <code>aggregatorMergeStrategy</code> also supports the <code>latest</code> and <code>earliest</code> strategies in addition to <code>strict</code> and <code>lenient</code> strategies from <code>lenientAggregatorMerge</code>.</p><p><a href="https://github.com/apache/druid/pull/14560" target="_blank" rel="noopener noreferrer">#14560</a> |
| <a href="https://github.com/apache/druid/pull/14598" target="_blank" rel="noopener noreferrer">#14598</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="broker-parallel-merge-config-options">Broker parallel merge config options<a href="#broker-parallel-merge-config-options" class="hash-link" aria-label="Direct link to Broker parallel merge config options" title="Direct link to Broker parallel merge config options"></a></h4><p>The paths for <code>druid.processing.merge.pool.*</code> and <code>druid.processing.merge.task.*</code> have been flattened to use <code>druid.processing.merge.*</code> instead. The legacy paths for the configs are now deprecated and will be removed in a future release. Migrate your settings to use the new paths because the old paths will be ignored in the future.</p><p><a href="https://github.com/apache/druid/pull/14695" target="_blank" rel="noopener noreferrer">#14695</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="ingestion-options-for-array-typed-columns">Ingestion options for ARRAY typed columns<a href="#ingestion-options-for-array-typed-columns" class="hash-link" aria-label="Direct link to Ingestion options for ARRAY typed columns" title="Direct link to Ingestion options for ARRAY typed columns"></a></h4><p>Starting with Druid 28.0.0, the MSQ task engine can detect and ingest arrays as ARRAY typed columns when you set the query context parameter <code>arrayIngestMode</code> to <code>array</code>. |
| The <code>arrayIngestMode</code> context parameter controls how ARRAY type values are stored in Druid segments.</p><p>When you set <code>arrayIngestMode</code> to <code>array</code> (recommended for SQL compliance), the MSQ task engine stores all ARRAY typed values in <a href="https://druid.apache.org/docs/latest/querying/arrays" target="_blank" rel="noopener noreferrer">ARRAY typed columns</a> and supports storing both VARCHAR and numeric typed arrays.</p><p>For backwards compatibility, <code>arrayIngestMode</code> defaults to <code>mvd</code>. When <code>"arrayIngestMode":"mvd"</code>, Druid only supports VARCHAR typed arrays and stores them as <a href="https://druid.apache.org/docs/latest/querying/multi-value-dimensions" target="_blank" rel="noopener noreferrer">multi-value string columns</a>.</p><p>When you set <code>arrayIngestMode</code> to <code>none</code>, Druid throws an exception when trying to store any type of arrays.</p><p>For more information on how to ingest <code>ARRAY</code> typed columns with SQL-based ingestion, see <a href="https://druid.apache.org/docs/latest/querying/sql-data-types#arrays" target="_blank" rel="noopener noreferrer">SQL data types</a> and <a href="https://druid.apache.org/docs/latest/querying/arrays" target="_blank" rel="noopener noreferrer">Array columns</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="incompatible-changes">Incompatible changes<a href="#incompatible-changes" class="hash-link" aria-label="Direct link to Incompatible changes" title="Direct link to Incompatible changes"></a></h3><h4 class="anchor anchorWithStickyNavbar_LWe7" id="removed-hadoop-2">Removed Hadoop 2<a href="#removed-hadoop-2" class="hash-link" aria-label="Direct link to Removed Hadoop 2" title="Direct link to Removed Hadoop 2"></a></h4><p>Support for Hadoop 2 has been removed. |
| Migrate to SQL-based ingestion or JSON-based batch ingestion if you are using Hadoop 2.x for ingestion today. |
| If migrating to Druid's built-in ingestion is not possible, you must upgrade your Hadoop infrastructure to 3.x+ before upgrading to Druid 28.0.0.</p><p><a href="https://github.com/apache/druid/pull/14763" target="_blank" rel="noopener noreferrer">#14763</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="removed-groupby-v1">Removed GroupBy v1<a href="#removed-groupby-v1" class="hash-link" aria-label="Direct link to Removed GroupBy v1" title="Direct link to Removed GroupBy v1"></a></h4><p>The GroupBy v1 engine has been removed. Use the GroupBy v2 engine instead, which has been the default GroupBy engine for several releases. |
| There should be no impact on your queries.</p><p>Additionally, <code>AggregatorFactory.getRequiredColumns</code> has been deprecated and will be removed in a future release. If you have an extension that implements <code>AggregatorFactory</code>, then this method should be removed from your implementation.</p><p><a href="https://github.com/apache/druid/pull/14866" target="_blank" rel="noopener noreferrer">#14866</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="removed-coordinator-dynamic-configs">Removed Coordinator dynamic configs<a href="#removed-coordinator-dynamic-configs" class="hash-link" aria-label="Direct link to Removed Coordinator dynamic configs" title="Direct link to Removed Coordinator dynamic configs"></a></h4><p>The <code>decommissioningMaxPercentOfMaxSegmentsToMove</code> config has been removed. |
| The use case for this config is handled by smart segment loading now, which is enabled by default.</p><p><a href="https://github.com/apache/druid/pull/14923" target="_blank" rel="noopener noreferrer">#14923</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="removed-cachingcost-strategy">Removed <code>cachingCost</code> strategy<a href="#removed-cachingcost-strategy" class="hash-link" aria-label="Direct link to removed-cachingcost-strategy" title="Direct link to removed-cachingcost-strategy"></a></h4><p>The <code>cachingCost</code> strategy for segment loading has been removed. |
| Use <code>cost</code> instead, which has the same benefits as <code>cachingCost</code>.</p><p>If you have <code>cachingCost</code> set, the system ignores this setting and automatically uses <code>cost</code>.</p><p><a href="https://github.com/apache/druid/pull/14798" target="_blank" rel="noopener noreferrer">#14798</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="removed-insertcannotorderbydescending">Removed <code>InsertCannotOrderByDescending</code><a href="#removed-insertcannotorderbydescending" class="hash-link" aria-label="Direct link to removed-insertcannotorderbydescending" title="Direct link to removed-insertcannotorderbydescending"></a></h4><p>The deprecated MSQ fault <code>InsertCannotOrderByDescending</code> has been removed.</p><p><a href="https://github.com/apache/druid/pull/14588" target="_blank" rel="noopener noreferrer">#14588</a></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="removed-the-backward-compatibility-code-for-the-handoff-api">Removed the backward compatibility code for the Handoff API<a href="#removed-the-backward-compatibility-code-for-the-handoff-api" class="hash-link" aria-label="Direct link to Removed the backward compatibility code for the Handoff API" title="Direct link to Removed the backward compatibility code for the Handoff API"></a></h4><p>The backward compatibility code for the Handoff API in <code>CoordinatorBasedSegmentHandoffNotifier</code> has been removed. |
| If you are upgrading from a Druid version older than 0.14.0, upgrade to a newer version of Druid before upgrading to Druid 28.0.0.</p><p><a href="https://github.com/apache/druid/pull/14652" target="_blank" rel="noopener noreferrer">#14652</a></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="developer-notes">Developer notes<a href="#developer-notes" class="hash-link" aria-label="Direct link to Developer notes" title="Direct link to Developer notes"></a></h3><h4 class="anchor anchorWithStickyNavbar_LWe7" id="dependency-updates">Dependency updates<a href="#dependency-updates" class="hash-link" aria-label="Direct link to Dependency updates" title="Direct link to Dependency updates"></a></h4><p>The following dependencies have had their versions bumped:</p><ul><li>Guava to <code>31.1-jre</code>. If you use an extension that has a transitive Guava dependency from Druid, it may be impacted <a href="https://github.com/apache/druid/pull/14767" target="_blank" rel="noopener noreferrer">#14767</a></li><li>Google Client APIs have been upgraded from 1.26.0 to 2.0.0 <a href="https://github.com/apache/druid/pull/14414" target="_blank" rel="noopener noreferrer">#14414</a></li><li>Apache Kafka has been upgraded to 3.5.1 <a href="https://github.com/apache/druid/pull/14721" target="_blank" rel="noopener noreferrer">#14721</a></li><li>Calcite has been upgraded to 1.35 <a href="https://github.com/apache/druid/pull/14510" target="_blank" rel="noopener noreferrer">#14510</a></li><li><code>RoaringBitmap</code> has been upgraded from 0.9.0 to 0.9.49 <a href="https://github.com/apache/druid/pull/15006" target="_blank" rel="noopener noreferrer">#15006</a></li><li><code>snappy-java</code> has been upgraded to 1.1.10.3 <a href="https://github.com/apache/druid/pull/14641" target="_blank" rel="noopener noreferrer">#14641</a></li><li><code>decode-uri-component</code> has been upgraded to 0.2.2 <a href="https://github.com/apache/druid/pull/13481" target="_blank" rel="noopener noreferrer">#13481</a></li><li><code>word-wrap</code> has been upgraded to 1.2.4 <a href="https://github.com/apache/druid/pull/14613" target="_blank" rel="noopener noreferrer">#14613</a></li><li><code>tough-cookie</code> has been upgraded to 4.1.3 <a href="https://github.com/apache/druid/pull/14557" target="_blank" rel="noopener noreferrer">#14557</a></li><li><code>qs</code> has been upgraded to 6.5.3 <a href="https://github.com/apache/druid/pull/13510" target="_blank" rel="noopener noreferrer">#13510</a></li><li><code>api-util</code> has been upgraded to 2.1.3 <a href="https://github.com/apache/druid/pull/14852" target="_blank" rel="noopener noreferrer">#14852</a></li><li><code>commons-cli</code> has been upgraded from 1.3.1 to 1.5.0 <a href="https://github.com/apache/druid/pull/14837" target="_blank" rel="noopener noreferrer">#14837</a></li><li><code>tukaani:xz</code> has been upgraded from 1.8 to 1.9 <a href="https://github.com/apache/druid/pull/14839" target="_blank" rel="noopener noreferrer">#14839</a></li><li><code>commons-compress</code> has been upgraded from 1.21 to 1.23.0 <a href="https://github.com/apache/druid/pull/14820" target="_blank" rel="noopener noreferrer">#14820</a></li><li><code>protobuf.version</code> has been upgraded from 3.21.7 to 3.24.0 <a href="https://github.com/apache/druid/pull/14823" target="_blank" rel="noopener noreferrer">#14823</a></li><li><code>dropwizard.metrics.version</code> has been upgraded from 4.0.0 to 4.2.19 <a href="https://github.com/apache/druid/pull/14824" target="_blank" rel="noopener noreferrer">#14824</a></li><li><code>assertj-core</code> has been upgraded from 3.19.0 to 3.24.2 <a href="https://github.com/apache/druid/pull/14815" target="_blank" rel="noopener noreferrer">#14815</a></li><li><code>maven-source-plugin</code> has been upgraded from 2.2.1 to 3.3.0 <a href="https://github.com/apache/druid/pull/14812" target="_blank" rel="noopener noreferrer">#14812</a></li><li><code>scala-library</code> has been upgraded from 2.13.9 to 2.13.11 <a href="https://github.com/apache/druid/pull/14826" target="_blank" rel="noopener noreferrer">#14826</a></li><li><code>oshi-core</code> has been upgraded from 6.4.2 to 6.4.4 <a href="https://github.com/apache/druid/pull/14814" target="_blank" rel="noopener noreferrer">#14814</a></li><li><code>maven-surefire-plugin</code> has been upgraded from 3.0.0-M7 to 3.1.2 <a href="https://github.com/apache/druid/pull/14813" target="_blank" rel="noopener noreferrer">#14813</a></li><li><code>apache-rat-plugin</code> has been upgraded from 0.12 to 0.15 <a href="https://github.com/apache/druid/pull/14817" target="_blank" rel="noopener noreferrer">#14817</a></li><li><code>jclouds.version</code> has been upgraded from 1.9.1 to 2.0.3 <a href="https://github.com/apache/druid/pull/14746" target="_blank" rel="noopener noreferrer">#14746</a></li><li><code>dropwizard.metrics:metrics-graphite</code> has been upgraded from 3.1.2 to 4.2.19 <a href="https://github.com/apache/druid/pull/14842" target="_blank" rel="noopener noreferrer">#14842</a></li><li><code>postgresql</code> has been upgraded from 42.4.1 to 42.6.0 <a href="https://github.com/apache/druid/pull/13959" target="_blank" rel="noopener noreferrer">#13959</a></li><li><code>org.mozilla:rhino</code> has been upgraded <a href="https://github.com/apache/druid/pull/14765" target="_blank" rel="noopener noreferrer">#14765</a></li><li><code>apache.curator.version</code> has been upgraded from 5.4.0 to 5.5.0 <a href="https://github.com/apache/druid/pull/14843" target="_blank" rel="noopener noreferrer">#14843</a></li><li><code>jackson-databind</code> has been upgraded to 2.12.7 <a href="https://github.com/apache/druid/pull/14770" target="_blank" rel="noopener noreferrer">#14770</a></li><li><code>icu4j</code> from 55.1 to 73.2 has been upgraded from 55.1 to 73.2 <a href="https://github.com/apache/druid/pull/14853" target="_blank" rel="noopener noreferrer">#14853</a></li><li><code>joda-time</code> has been upgraded from 2.12.4 to 2.12.5 <a href="https://github.com/apache/druid/pull/14855" target="_blank" rel="noopener noreferrer">#14855</a></li><li><code>tough-cookie</code> has been upgraded from 4.0.0 to 4.1.3 <a href="https://github.com/apache/druid/pull/14557" target="_blank" rel="noopener noreferrer">#14557</a></li><li><code>word-wrap</code> has been upgraded from 1.2.3 to 1.2.4 <a href="https://github.com/apache/druid/pull/14613" target="_blank" rel="noopener noreferrer">#14613</a></li><li><code>decode-uri-component</code> has been upgraded from 0.2.0 to 0.2.2 <a href="https://github.com/apache/druid/pull/13481" target="_blank" rel="noopener noreferrer">#13481</a></li><li><code>snappy-java</code> has been upgraded from 1.1.10.1 to 1.1.10.3 <a href="https://github.com/apache/druid/pull/14641" target="_blank" rel="noopener noreferrer">#14641</a></li><li>Hibernate validator version has been upgraded <a href="https://github.com/apache/druid/pull/14757" target="_blank" rel="noopener noreferrer">#14757</a></li><li>The Dependabot PR limit for Java dependencies has been increased <a href="https://github.com/apache/druid/pull/14804" target="_blank" rel="noopener noreferrer">#14804</a></li><li><code>jetty</code> has been upgraded from 9.4.51.v20230217 to 9.4.53.v20231009 <a href="https://github.com/apache/druid/pull/15129" target="_blank" rel="noopener noreferrer">#15129</a></li><li><code>netty4</code> has been upgraded from 4.1.94.Final to 4.1.100.Final <a href="https://github.com/apache/druid/pull/15129" target="_blank" rel="noopener noreferrer">#15129</a></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="credits">Credits<a href="#credits" class="hash-link" aria-label="Direct link to Credits" title="Direct link to Credits"></a></h2><p>Thanks to everyone who contributed to this release!</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/docs/latest/misc/papers-and-talks"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Papers</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/docs/latest/release-info/upgrade-notes"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Upgrade notes</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#druid-2801" class="table-of-contents__link toc-highlight">Druid 28.0.1</a><ul><li><a href="#bug-fixes" class="table-of-contents__link toc-highlight">Bug fixes</a></li></ul></li><li><a href="#druid-2800" class="table-of-contents__link toc-highlight">Druid 28.0.0</a></li><li><a href="#important-features-changes-and-deprecations" class="table-of-contents__link toc-highlight">Important features, changes, and deprecations</a><ul><li><a href="#sql-compatibility" class="table-of-contents__link toc-highlight">SQL compatibility</a></li><li><a href="#sql-planner-improvements" class="table-of-contents__link toc-highlight">SQL planner improvements</a></li><li><a href="#async-query-and-query-from-deep-storage" class="table-of-contents__link toc-highlight">Async query and query from deep storage</a></li><li><a href="#msq-queries-for-realtime-tasks" class="table-of-contents__link toc-highlight">MSQ queries for realtime tasks</a></li><li><a href="#msq-support-for-union-all-queries" class="table-of-contents__link toc-highlight">MSQ support for UNION ALL queries</a></li><li><a href="#ingest-from-multiple-kafka-topics-to-a-single-datasource" class="table-of-contents__link toc-highlight">Ingest from multiple Kafka topics to a single datasource</a></li><li><a href="#sql-unnest-and-ingestion-flattening" class="table-of-contents__link toc-highlight">SQL UNNEST and ingestion flattening</a></li><li><a href="#window-functions-experimental" class="table-of-contents__link toc-highlight">Window functions (experimental)</a></li><li><a href="#concurrent-append-and-replace-experimental" class="table-of-contents__link toc-highlight">Concurrent append and replace (experimental)</a></li></ul></li><li><a href="#functional-area-and-related-changes" class="table-of-contents__link toc-highlight">Functional area and related changes</a><ul><li><a href="#web-console" class="table-of-contents__link toc-highlight">Web console</a></li><li><a href="#ingestion" class="table-of-contents__link toc-highlight">Ingestion</a></li><li><a href="#querying" class="table-of-contents__link toc-highlight">Querying</a></li><li><a href="#cluster-management" class="table-of-contents__link toc-highlight">Cluster management</a></li><li><a href="#data-management" class="table-of-contents__link toc-highlight">Data management</a></li><li><a href="#metrics-and-monitoring" class="table-of-contents__link toc-highlight">Metrics and monitoring</a></li><li><a href="#extensions" class="table-of-contents__link toc-highlight">Extensions</a></li><li><a href="#documentation-improvements" class="table-of-contents__link toc-highlight">Documentation improvements</a></li></ul></li><li><a href="#upgrade-notes-and-incompatible-changes" class="table-of-contents__link toc-highlight">Upgrade notes and incompatible changes</a><ul><li><a href="#upgrade-notes" class="table-of-contents__link toc-highlight">Upgrade notes</a></li><li><a href="#incompatible-changes" class="table-of-contents__link toc-highlight">Incompatible changes</a></li><li><a href="#developer-notes" class="table-of-contents__link toc-highlight">Developer notes</a></li></ul></li><li><a href="#credits" class="table-of-contents__link toc-highlight">Credits</a></li></ul></div></div></div></div></main></div></div><footer class="footer"><div class="container container-fluid"><div class="footer__bottom text--center"><div class="margin-bottom--sm"><img src="/img/favicon.png" class="themedImage_ToTc themedImage--light_HNdA footer__logo"><img src="/img/favicon.png" class="themedImage_ToTc themedImage--dark_i4oU footer__logo"></div><div class="footer__copyright">Copyright © 2023 Apache Software Foundation. Except where otherwise noted, licensed under CC BY-SA 4.0. Apache Druid, Druid, and the Druid logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</div></div></div></footer></div> |
| <script src="/assets/js/runtime~main.91da3985.js"></script> |
| <script src="/assets/js/main.1f0e5e69.js"></script> |
| </body> |
| </html> |
