| <!doctype html> |
| <html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-current plugin-docs plugin-id-default docs-doc-id-multi-stage-query/reference"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="generator" content="Docusaurus v2.4.1"> |
| <title data-rh="true">SQL-based ingestion reference | Apache® Druid</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:image" content="https://druid.apache.org/img/druid_nav.png"><meta data-rh="true" name="twitter:image" content="https://druid.apache.org/img/druid_nav.png"><meta data-rh="true" property="og:url" content="https://druid.apache.org/docs/26.0.0/multi-stage-query/reference"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="current"><meta data-rh="true" name="docusaurus_tag" content="docs-default-current"><meta data-rh="true" name="docsearch:version" content="current"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-current"><meta data-rh="true" property="og:title" content="SQL-based ingestion reference | Apache® Druid"><meta data-rh="true" name="description" content="<!--"><meta data-rh="true" property="og:description" content="<!--"><link data-rh="true" rel="icon" href="/img/favicon.png"><link data-rh="true" rel="canonical" href="https://druid.apache.org/docs/26.0.0/multi-stage-query/reference"><link data-rh="true" rel="alternate" href="https://druid.apache.org/docs/26.0.0/multi-stage-query/reference" hreflang="en"><link data-rh="true" rel="alternate" href="https://druid.apache.org/docs/26.0.0/multi-stage-query/reference" hreflang="x-default"><link rel="preconnect" href="https://www.google-analytics.com"> |
| <link rel="preconnect" href="https://www.googletagmanager.com"> |
| <script async src="https://www.googletagmanager.com/gtag/js?id=UA-131010415-1"></script> |
| <script>function gtag(){dataLayer.push(arguments)}window.dataLayer=window.dataLayer||[],gtag("js",new Date),gtag("config","UA-131010415-1",{})</script> |
| |
| |
| |
| |
| <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"> |
| <script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><link rel="stylesheet" href="/assets/css/styles.f80751b3.css"> |
| <link rel="preload" href="/assets/js/runtime~main.38900cbf.js" as="script"> |
| <link rel="preload" href="/assets/js/main.5e106d68.js" as="script"> |
| </head> |
| <body class="navigation-with-keyboard"> |
| <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus"> |
| <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><nav aria-label="Main" class="navbar navbar--fixed-top navbar--dark"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/druid_nav.png" alt="Apache® Druid" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/druid_nav.png" alt="Apache® Druid" class="themedImage_ToTc themedImage--dark_i4oU"></div></a></div><div class="navbar__items navbar__items--right"><a class="navbar__item navbar__link" href="/technology">Technology</a><a class="navbar__item navbar__link" href="/use-cases">Use Cases</a><a class="navbar__item navbar__link" href="/druid-powered">Powered By</a><a class="navbar__item navbar__link" href="/docs/26.0.0/design/">Docs</a><a class="navbar__item navbar__link" href="/community/">Community</a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Apache®</a><ul class="dropdown__menu"><li><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Foundation<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://apachecon.com/?ref=druid.apache.org" target="_blank" rel="noopener noreferrer" class="dropdown__link">Events<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/licenses/" target="_blank" rel="noopener noreferrer" class="dropdown__link">License<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Thanks<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/security/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Security<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Sponsorship<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a class="navbar__item navbar__link" href="/downloads/">Download</a><div class="searchBox_ZlJk"><div class="navbar__search"><span aria-label="expand searchbar" role="button" class="search-icon" tabindex="0"></span><input type="search" id="search_input_react" placeholder="Loading..." aria-label="Search" class="navbar__search-input search-bar" disabled=""></div></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/26.0.0/design/">Getting started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/26.0.0/tutorials/tutorial-batch">Tutorials</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/26.0.0/design/architecture">Design</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/docs/26.0.0/ingestion/">Ingestion</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/ingestion/">Ingestion</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/ingestion/data-formats">Data formats</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/ingestion/data-model">Data model</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/ingestion/rollup">Data rollup</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/ingestion/partitioning">Partitioning</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/ingestion/ingestion-spec">Ingestion spec</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/ingestion/schema-design">Schema design tips</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/26.0.0/development/extensions-core/kafka-ingestion">Stream ingestion</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/26.0.0/ingestion/native-batch">Batch ingestion</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/docs/26.0.0/multi-stage-query/">SQL-based ingestion 🆕</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/multi-stage-query/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/multi-stage-query/concepts">Key concepts</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/multi-stage-query/api">API</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/multi-stage-query/security">Security</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/multi-stage-query/examples">Examples</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/docs/26.0.0/multi-stage-query/reference">Reference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/multi-stage-query/known-issues">Known issues</a></li></ul></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/ingestion/tasks">Task reference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/ingestion/faq">Troubleshooting FAQ</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/26.0.0/data-management/">Data management</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/26.0.0/querying/sql">Querying</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/26.0.0/configuration/">Configuration</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/26.0.0/operations/web-console">Operations</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/26.0.0/development/overview">Development</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/26.0.0/misc/papers-and-talks">Misc</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Ingestion</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">SQL-based ingestion 🆕</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Reference</span><meta itemprop="position" content="3"></li></ul></nav><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>SQL-based ingestion reference</h1></header><blockquote><p>This page describes SQL-based batch ingestion using the <a href="/docs/26.0.0/multi-stage-query/"><code>druid-multi-stage-query</code></a> |
| extension, new in Druid 24.0. Refer to the <a href="/docs/26.0.0/ingestion/#batch">ingestion methods</a> table to determine which |
| ingestion method is right for you.</p></blockquote><h2 class="anchor anchorWithStickyNavbar_LWe7" id="sql-reference">SQL reference<a href="#sql-reference" class="hash-link" aria-label="Direct link to SQL reference" title="Direct link to SQL reference"></a></h2><p>This topic is a reference guide for the multi-stage query architecture in Apache Druid. For examples of real-world |
| usage, refer to the <a href="/docs/26.0.0/multi-stage-query/examples">Examples</a> page.</p><p><code>INSERT</code> and <code>REPLACE</code> load data into a Druid datasource from either an external input source, or from another |
| datasource. When loading from an external datasource, you typically must provide the kind of input source, |
| the data format, and the schema (signature) of the input file. Druid provides <em>table functions</em> to allow you to |
| specify the external file. There are two kinds. <code>EXTERN</code> works with the JSON-serialized specs for the three |
| items, using the same JSON you would use in native ingest. A set of other, input-source-specific functions |
| use SQL syntax to specify the format and the input schema. There is one function for each input source. The |
| input-source-specific functions allow you to use SQL query parameters to specify the set of files (or URIs), |
| making it easy to reuse the same SQL statement for each ingest: just specify the set of files to use each time.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="extern-function"><code>EXTERN</code> Function<a href="#extern-function" class="hash-link" aria-label="Direct link to extern-function" title="Direct link to extern-function"></a></h3><p>Use the <code>EXTERN</code> function to read external data. The function has two variations.</p><p>Function variation 1, with the input schema expressed as JSON:</p><div class="language-sql codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sql codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token keyword" style="font-style:italic">SELECT</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token operator" style="color:rgb(137, 221, 255)"><</span><span class="token keyword" style="font-style:italic">column</span><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token keyword" style="font-style:italic">FROM</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">TABLE</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> EXTERN</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">'<Druid input source>'</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">'<Druid input format>'</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">'<row signature>'</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><code>EXTERN</code> consists of the following parts:</p><ol><li>Any <a href="/docs/26.0.0/ingestion/native-batch-input-sources">Druid input source</a> as a JSON-encoded string.</li><li>Any <a href="/docs/26.0.0/ingestion/data-formats">Druid input format</a> as a JSON-encoded string.</li><li>A row signature, as a JSON-encoded array of column descriptors. Each column descriptor must have a |
| <code>name</code> and a <code>type</code>. The type can be <code>string</code>, <code>long</code>, <code>double</code>, or <code>float</code>. This row signature is |
| used to map the external data into the SQL layer.</li></ol><p>Variation 2, with the input schema expressed in SQL using an <code>EXTEND</code> clause. (See the next |
| section for more detail on <code>EXTEND</code>). This format also uses named arguments to make the |
| SQL a bit easier to read:</p><div class="language-sql codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sql codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token keyword" style="font-style:italic">SELECT</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token operator" style="color:rgb(137, 221, 255)"><</span><span class="token keyword" style="font-style:italic">column</span><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token keyword" style="font-style:italic">FROM</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">TABLE</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> EXTERN</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> inputSource </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">'<Druid input source>'</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> inputFormat </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">'<Druid input format>'</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token operator" style="color:rgb(137, 221, 255)"><</span><span class="token keyword" style="font-style:italic">columns</span><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The input source and format are as above. The columns are expressed as in a SQL <code>CREATE TABLE</code>. |
| Example: <code>(timestamp VARCHAR, metricType VARCHAR, value BIGINT)</code>. The optional <code>EXTEND</code> keyword |
| can precede the column list: <code>EXTEND (timestamp VARCHAR...)</code>.</p><p>For more information, see <a href="/docs/26.0.0/multi-stage-query/concepts#extern">Read external data with EXTERN</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="insert"><code>INSERT</code><a href="#insert" class="hash-link" aria-label="Direct link to insert" title="Direct link to insert"></a></h3><p>Use the <code>INSERT</code> statement to insert data.</p><p>Unlike standard SQL, <code>INSERT</code> loads data into the target table according to column name, not positionally. If necessary, |
| use <code>AS</code> in your <code>SELECT</code> column list to assign the correct names. Do not rely on their positions within the SELECT |
| clause.</p><p>Statement format:</p><div class="language-sql codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sql codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token keyword" style="font-style:italic">INSERT</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">INTO</span><span class="token plain"> </span><span class="token operator" style="color:rgb(137, 221, 255)"><</span><span class="token keyword" style="font-style:italic">table</span><span class="token plain"> name</span><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token operator" style="color:rgb(137, 221, 255)"><</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">SELECT</span><span class="token plain"> query </span><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">PARTITIONED </span><span class="token keyword" style="font-style:italic">BY</span><span class="token plain"> </span><span class="token operator" style="color:rgb(137, 221, 255)"><</span><span class="token keyword" style="font-style:italic">time</span><span class="token plain"> frame</span><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">CLUSTERED</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">BY</span><span class="token plain"> </span><span class="token operator" style="color:rgb(137, 221, 255)"><</span><span class="token keyword" style="font-style:italic">column</span><span class="token plain"> list</span><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>INSERT consists of the following parts:</p><ol><li>Optional <a href="/docs/26.0.0/multi-stage-query/reference#context-parameters">context parameters</a>.</li><li>An <code>INSERT INTO <dataSource></code> clause at the start of your query, such as <code>INSERT INTO your-table</code>.</li><li>A clause for the data you want to insert, such as <code>SELECT ... FROM ...</code>. You can use <a href="#extern-function"><code>EXTERN</code></a> |
| to reference external tables using <code>FROM TABLE(EXTERN(...))</code>.</li><li>A <a href="#partitioned-by">PARTITIONED BY</a> clause, such as <code>PARTITIONED BY DAY</code>.</li><li>An optional <a href="#clustered-by">CLUSTERED BY</a> clause.</li></ol><p>For more information, see <a href="/docs/26.0.0/multi-stage-query/concepts#insert">Load data with INSERT</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="replace"><code>REPLACE</code><a href="#replace" class="hash-link" aria-label="Direct link to replace" title="Direct link to replace"></a></h3><p>You can use the <code>REPLACE</code> function to replace all or some of the data.</p><p>Unlike standard SQL, <code>REPLACE</code> loads data into the target table according to column name, not positionally. If necessary, |
| use <code>AS</code> in your <code>SELECT</code> column list to assign the correct names. Do not rely on their positions within the SELECT |
| clause.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="replace-all-data"><code>REPLACE</code> all data<a href="#replace-all-data" class="hash-link" aria-label="Direct link to replace-all-data" title="Direct link to replace-all-data"></a></h4><p>Function format to replace all data:</p><div class="language-sql codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sql codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token keyword" style="font-style:italic">REPLACE</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">INTO</span><span class="token plain"> </span><span class="token operator" style="color:rgb(137, 221, 255)"><</span><span class="token plain">target </span><span class="token keyword" style="font-style:italic">table</span><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">OVERWRITE </span><span class="token keyword" style="font-style:italic">ALL</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token operator" style="color:rgb(137, 221, 255)"><</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">SELECT</span><span class="token plain"> query </span><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">PARTITIONED </span><span class="token keyword" style="font-style:italic">BY</span><span class="token plain"> </span><span class="token operator" style="color:rgb(137, 221, 255)"><</span><span class="token keyword" style="font-style:italic">time</span><span class="token plain"> granularity</span><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">CLUSTERED</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">BY</span><span class="token plain"> </span><span class="token operator" style="color:rgb(137, 221, 255)"><</span><span class="token keyword" style="font-style:italic">column</span><span class="token plain"> list</span><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="replace-specific-time-ranges"><code>REPLACE</code> specific time ranges<a href="#replace-specific-time-ranges" class="hash-link" aria-label="Direct link to replace-specific-time-ranges" title="Direct link to replace-specific-time-ranges"></a></h4><p>Function format to replace specific time ranges:</p><div class="language-sql codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sql codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token keyword" style="font-style:italic">REPLACE</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">INTO</span><span class="token plain"> </span><span class="token operator" style="color:rgb(137, 221, 255)"><</span><span class="token plain">target </span><span class="token keyword" style="font-style:italic">table</span><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">OVERWRITE </span><span class="token keyword" style="font-style:italic">WHERE</span><span class="token plain"> __time </span><span class="token operator" style="color:rgb(137, 221, 255)">>=</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">TIMESTAMP</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">'<lower bound>'</span><span class="token plain"> </span><span class="token operator" style="color:rgb(137, 221, 255)">AND</span><span class="token plain"> __time </span><span class="token operator" style="color:rgb(137, 221, 255)"><</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">TIMESTAMP</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">'<upper bound>'</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token operator" style="color:rgb(137, 221, 255)"><</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">SELECT</span><span class="token plain"> query </span><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">PARTITIONED </span><span class="token keyword" style="font-style:italic">BY</span><span class="token plain"> </span><span class="token operator" style="color:rgb(137, 221, 255)"><</span><span class="token keyword" style="font-style:italic">time</span><span class="token plain"> granularity</span><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">CLUSTERED</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">BY</span><span class="token plain"> </span><span class="token operator" style="color:rgb(137, 221, 255)"><</span><span class="token keyword" style="font-style:italic">column</span><span class="token plain"> list</span><span class="token operator" style="color:rgb(137, 221, 255)">></span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><code>REPLACE</code> consists of the following parts:</p><ol><li>Optional <a href="/docs/26.0.0/multi-stage-query/reference#context-parameters">context parameters</a>.</li><li>A <code>REPLACE INTO <dataSource></code> clause at the start of your query, such as <code>REPLACE INTO "your-table".</code></li><li>An OVERWRITE clause after the datasource, either OVERWRITE ALL or OVERWRITE WHERE:<ul><li>OVERWRITE ALL replaces the entire existing datasource with the results of the query.</li><li>OVERWRITE WHERE drops the time segments that match the condition you set. Conditions are based on the <code>__time</code> |
| column and use the format <code>__time [< > = <= >=] TIMESTAMP</code>. Use them with AND, OR, and NOT between them, inclusive |
| of the timestamps specified. No other expressions or functions are valid in OVERWRITE.</li></ul></li><li>A clause for the actual data you want to use for the replacement.</li><li>A <a href="#partitioned-by">PARTITIONED BY</a> clause, such as <code>PARTITIONED BY DAY</code>.</li><li>An optional <a href="#clustered-by">CLUSTERED BY</a> clause.</li></ol><p>For more information, see <a href="/docs/26.0.0/multi-stage-query/concepts#replace">Overwrite data with REPLACE</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="partitioned-by"><code>PARTITIONED BY</code><a href="#partitioned-by" class="hash-link" aria-label="Direct link to partitioned-by" title="Direct link to partitioned-by"></a></h3><p>The <code>PARTITIONED BY <time granularity></code> clause is required for <a href="#insert">INSERT</a> and <a href="#replace">REPLACE</a>. See |
| <a href="/docs/26.0.0/multi-stage-query/concepts#partitioning">Partitioning</a> for details.</p><p>The following granularity arguments are accepted:</p><ul><li>Time unit keywords: <code>HOUR</code>, <code>DAY</code>, <code>MONTH</code>, or <code>YEAR</code>. Equivalent to <code>FLOOR(__time TO TimeUnit)</code>.</li><li>Time units as ISO 8601 period strings: :<code>'PT1H'</code>, '<code>P1D</code>, etc. (Druid 26.0 and later.)</li><li><code>TIME_FLOOR(__time, 'granularity_string')</code>, where granularity_string is one of the ISO 8601 periods listed below. The |
| first argument must be <code>__time</code>.</li><li><code>FLOOR(__time TO TimeUnit)</code>, where <code>TimeUnit</code> is any unit supported by the <a href="/docs/26.0.0/querying/sql-scalar#date-and-time-functions">FLOOR function</a>. The first argument must be <code>__time</code>.</li><li><code>ALL</code> or <code>ALL TIME</code>, which effectively disables time partitioning by placing all data in a single time chunk. To use |
| LIMIT or OFFSET at the outer level of your <code>INSERT</code> or <code>REPLACE</code> query, you must set <code>PARTITIONED BY</code> to <code>ALL</code> or <code>ALL TIME</code>.</li></ul><p>Earlier versions required the <code>TIME_FLOOR</code> notation to specify a granularity other than the keywords. |
| In the current version, the string constant provides a simpler equivalent solution.</p><p>The following ISO 8601 periods are supported for <code>TIME_FLOOR</code> and the string constant:</p><ul><li>PT1S</li><li>PT1M</li><li>PT5M</li><li>PT10M</li><li>PT15M</li><li>PT30M</li><li>PT1H</li><li>PT6H</li><li>P1D</li><li>P1W</li><li>P1M</li><li>P3M</li><li>P1Y</li></ul><p>For more information about partitioning, see <a href="/docs/26.0.0/multi-stage-query/concepts#partitioning">Partitioning</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="clustered-by"><code>CLUSTERED BY</code><a href="#clustered-by" class="hash-link" aria-label="Direct link to clustered-by" title="Direct link to clustered-by"></a></h3><p>The <code>CLUSTERED BY <column list></code> clause is optional for <a href="#insert">INSERT</a> and <a href="#replace">REPLACE</a>. It accepts a list of |
| column names or expressions.</p><p>For more information about clustering, see <a href="/docs/26.0.0/multi-stage-query/concepts#clustering">Clustering</a>.</p><a name="context"></a><h2 class="anchor anchorWithStickyNavbar_LWe7" id="context-parameters">Context parameters<a href="#context-parameters" class="hash-link" aria-label="Direct link to Context parameters" title="Direct link to Context parameters"></a></h2><p>In addition to the Druid SQL <a href="/docs/26.0.0/querying/sql-query-context">context parameters</a>, the multi-stage query task engine accepts certain context parameters that are specific to it.</p><p>Use context parameters alongside your queries to customize the behavior of the query. If you're using the API, include the context parameters in the query context when you submit a query:</p><div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"query"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"SELECT 1 + 1"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"context"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"<key>"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"<value>"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"maxNumTasks"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">3</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>If you're using the web console, you can specify the context parameters through various UI options.</p><p>The following table lists the context parameters for the MSQ task engine:</p><table><thead><tr><th>Parameter</th><th>Description</th><th>Default value</th></tr></thead><tbody><tr><td><code>maxNumTasks</code></td><td>SELECT, INSERT, REPLACE<br><br>The maximum total number of tasks to launch, including the controller task. The lowest possible value for this setting is 2: one controller and one worker. All tasks must be able to launch simultaneously. If they cannot, the query returns a <code>TaskStartTimeout</code> error code after approximately 10 minutes.<br><br>May also be provided as <code>numTasks</code>. If both are present, <code>maxNumTasks</code> takes priority.</td><td>2</td></tr><tr><td><code>taskAssignment</code></td><td>SELECT, INSERT, REPLACE<br><br>Determines how many tasks to use. Possible values include: <ul><li><code>max</code>: Uses as many tasks as possible, up to <code>maxNumTasks</code>.</li><li><code>auto</code>: When file sizes can be determined through directory listing (for example: local files, S3, GCS, HDFS) uses as few tasks as possible without exceeding 10 GiB or 10,000 files per task, unless exceeding these limits is necessary to stay within <code>maxNumTasks</code>. When file sizes cannot be determined through directory listing (for example: http), behaves the same as <code>max</code>.</li></ul></td><td><code>max</code></td></tr><tr><td><code>finalizeAggregations</code></td><td>SELECT, INSERT, REPLACE<br><br>Determines the type of aggregation to return. If true, Druid finalizes the results of complex aggregations that directly appear in query results. If false, Druid returns the aggregation's intermediate type rather than finalized type. This parameter is useful during ingestion, where it enables storing sketches directly in Druid tables. For more information about aggregations, see <a href="/docs/26.0.0/querying/sql-aggregations">SQL aggregation functions</a>.</td><td>true</td></tr><tr><td><code>sqlJoinAlgorithm</code></td><td>SELECT, INSERT, REPLACE<br><br>Algorithm to use for JOIN. Use <code>broadcast</code> (the default) for broadcast hash join or <code>sortMerge</code> for sort-merge join. Affects all JOIN operations in the query. See <a href="#joins">Joins</a> for more details.</td><td><code>broadcast</code></td></tr><tr><td><code>rowsInMemory</code></td><td>INSERT or REPLACE<br><br>Maximum number of rows to store in memory at once before flushing to disk during the segment generation process. Ignored for non-INSERT queries. In most cases, use the default value. You may need to override the default if you run into one of the <a href="/docs/26.0.0/multi-stage-query/known-issues">known issues</a> around memory usage.</td><td>100,000</td></tr><tr><td><code>segmentSortOrder</code></td><td>INSERT or REPLACE<br><br>Normally, Druid sorts rows in individual segments using <code>__time</code> first, followed by the <a href="#clustered-by">CLUSTERED BY</a> clause. When you set <code>segmentSortOrder</code>, Druid sorts rows in segments using this column list first, followed by the CLUSTERED BY order.<br><br>You provide the column list as comma-separated values or as a JSON array in string form. If your query includes <code>__time</code>, then this list must begin with <code>__time</code>. For example, consider an INSERT query that uses <code>CLUSTERED BY country</code> and has <code>segmentSortOrder</code> set to <code>__time,city</code>. Within each time chunk, Druid assigns rows to segments based on <code>country</code>, and then within each of those segments, Druid sorts those rows by <code>__time</code> first, then <code>city</code>, then <code>country</code>.</td><td>empty list</td></tr><tr><td><code>maxParseExceptions</code></td><td>SELECT, INSERT, REPLACE<br><br>Maximum number of parse exceptions that are ignored while executing the query before it stops with <code>TooManyWarningsFault</code>. To ignore all the parse exceptions, set the value to -1.</td><td>0</td></tr><tr><td><code>rowsPerSegment</code></td><td>INSERT or REPLACE<br><br>The number of rows per segment to target. The actual number of rows per segment may be somewhat higher or lower than this number. In most cases, use the default. For general information about sizing rows per segment, see <a href="/docs/26.0.0/operations/segment-optimization">Segment Size Optimization</a>.</td><td>3,000,000</td></tr><tr><td><code>indexSpec</code></td><td>INSERT or REPLACE<br><br>An <a href="/docs/26.0.0/ingestion/ingestion-spec#indexspec"><code>indexSpec</code></a> to use when generating segments. May be a JSON string or object. See <a href="/docs/26.0.0/ingestion/ingestion-spec#front-coding">Front coding</a> for details on configuring an <code>indexSpec</code> with front coding.</td><td>See <a href="/docs/26.0.0/ingestion/ingestion-spec#indexspec"><code>indexSpec</code></a>.</td></tr><tr><td><code>durableShuffleStorage</code></td><td>SELECT, INSERT, REPLACE <br><br>Whether to use durable storage for shuffle mesh. To use this feature, configure the durable storage at the server level using <code>druid.msq.intermediate.storage.enable=true</code>). If these properties are not configured, any query with the context variable <code>durableShuffleStorage=true</code> fails with a configuration error. <br><br></td><td><code>false</code></td></tr><tr><td><code>faultTolerance</code></td><td>SELECT, INSERT, REPLACE<br><br> Whether to turn on fault tolerance mode or not. Failed workers are retried based on <a href="#limits">Limits</a>. Cannot be used when <code>durableShuffleStorage</code> is explicitly set to false.</td><td><code>false</code></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="joins">Joins<a href="#joins" class="hash-link" aria-label="Direct link to Joins" title="Direct link to Joins"></a></h2><p>Joins in multi-stage queries use one of two algorithms, based on the <a href="#context-parameters">context parameter</a> |
| <code>sqlJoinAlgorithm</code>. This context parameter applies to the entire SQL statement, so it is not possible to mix different |
| join algorithms in the same query.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="broadcast">Broadcast<a href="#broadcast" class="hash-link" aria-label="Direct link to Broadcast" title="Direct link to Broadcast"></a></h3><p>Set <code>sqlJoinAlgorithm</code> to <code>broadcast</code>.</p><p>The default join algorithm for multi-stage queries is a broadcast hash join, which is similar to how |
| <a href="/docs/26.0.0/querying/query-execution#join">joins are executed with native queries</a>. First, any adjacent joins are flattened |
| into a structure with a "base" input (the bottom-leftmost one) and other leaf inputs (the rest). Next, any subqueries |
| that are inputs the join (either base or other leafs) are planned into independent stages. Then, the non-base leaf |
| inputs are all connected as broadcast inputs to the "base" stage.</p><p>Together, all of these non-base leaf inputs must not exceed the <a href="#limits">limit on broadcast table footprint</a>. There |
| is no limit on the size of the base (leftmost) input.</p><p>Only LEFT JOIN, INNER JOIN, and CROSS JOIN are supported with with <code>broadcast</code>.</p><p>Join conditions, if present, must be equalities. It is not necessary to include a join condition; for example, |
| <code>CROSS JOIN</code> and comma join do not require join conditions.</p><p>As an example, the following statement has a single join chain where <code>orders</code> is the base input, and <code>products</code> and |
| <code>customers</code> are non-base leaf inputs. The query will first read <code>products</code> and <code>customers</code>, then broadcast both to |
| the stage that reads <code>orders</code>. That stage loads the broadcast inputs (<code>products</code> and <code>customers</code>) in memory, and walks |
| through <code>orders</code> row by row. The results are then aggregated and written to the table <code>orders_enriched</code>. The broadcast |
| inputs (<code>products</code> and <code>customers</code>) must fall under the limit on broadcast table footprint, but the base <code>orders</code> input |
| can be unlimited in size.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">REPLACE INTO orders_enriched</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">OVERWRITE ALL</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">SELECT</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> orders.__time,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> products.name AS product_name,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> customers.name AS customer_name,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> SUM(orders.amount) AS amount</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">FROM orders</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">LEFT JOIN products ON orders.product_id = products.id</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">LEFT JOIN customers ON orders.customer_id = customers.id</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">GROUP BY 1, 2</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">PARTITIONED BY HOUR</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">CLUSTERED BY product_name</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sort-merge">Sort-merge<a href="#sort-merge" class="hash-link" aria-label="Direct link to Sort-merge" title="Direct link to Sort-merge"></a></h3><p>Set <code>sqlJoinAlgorithm</code> to <code>sortMerge</code>.</p><p>Multi-stage queries can use a sort-merge join algorithm. With this algorithm, each pairwise join is planned into its own |
| stage with two inputs. The two inputs are partitioned and sorted using a hash partitioning on the same key. This |
| approach is generally less performant, but more scalable, than <code>broadcast</code>. There are various scenarios where broadcast |
| join would return a <a href="#errors"><code>BroadcastTablesTooLarge</code></a> error, but a sort-merge join would succeed.</p><p>There is no limit on the overall size of either input, so sort-merge is a good choice for performing a join of two large |
| inputs, or for performing a self-join of a large input with itself.</p><p>There is a limit on the amount of data associated with each individual key. If <em>both</em> sides of the join exceed this |
| limit, the query returns a <a href="#errors"><code>TooManyRowsWithSameKey</code></a> error. If only one side exceeds the limit, the query |
| does not return this error.</p><p>Join conditions, if present, must be equalities. It is not necessary to include a join condition; for example, |
| <code>CROSS JOIN</code> and comma join do not require join conditions.</p><p>All join types are supported with <code>sortMerge</code>: LEFT, RIGHT, INNER, FULL, and CROSS.</p><p>As an example, the following statement runs using a single sort-merge join stage that receives <code>eventstream</code> |
| (partitioned on <code>user_id</code>) and <code>users</code> (partitioned on <code>id</code>) as inputs. There is no limit on the size of either input.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">REPLACE INTO eventstream_enriched</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">OVERWRITE ALL</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">SELECT</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> eventstream.__time,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> eventstream.user_id,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> eventstream.event_type,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> eventstream.event_details,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> users.signup_date AS user_signup_date</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">FROM eventstream</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">LEFT JOIN users ON eventstream.user_id = users.id</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">PARTITIONED BY HOUR</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">CLUSTERED BY user</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="durable-storage">Durable Storage<a href="#durable-storage" class="hash-link" aria-label="Direct link to Durable Storage" title="Direct link to Durable Storage"></a></h2><p>Using durable storage with your SQL-based ingestions can improve their reliability by writing intermediate files to a storage location temporarily. </p><p>To prevent durable storage from getting filled up with temporary files in case the tasks fail to clean them up, a periodic |
| cleaner can be scheduled to clean the directories corresponding to which there isn't a controller task running. It utilizes |
| the storage connector to work upon the durable storage. The durable storage location should only be utilized to store the output |
| for cluster's MSQ tasks. If the location contains other files or directories, then they will get cleaned up as well.</p><p>Enabling durable storage also enables the use of local disk to store temporary files, such as the intermediate files produced |
| by the super sorter. The limit set by <code>druid.indexer.task.tmpStorageBytesPerTask</code> for maximum number of bytes of local |
| storage to be used per task will be respected by MSQ tasks. If the configured limit is too low, <code>NotEnoughTemporaryStorageFault</code> |
| may be thrown.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="enable-durable-storage">Enable durable storage<a href="#enable-durable-storage" class="hash-link" aria-label="Direct link to Enable durable storage" title="Direct link to Enable durable storage"></a></h3><p>To enable durable storage, you need to set the following common service properties:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.msq.intermediate.storage.enable=true</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.msq.intermediate.storage.type=s3</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.msq.intermediate.storage.bucket=YOUR_BUCKET</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.msq.intermediate.storage.prefix=YOUR_PREFIX</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">druid.msq.intermediate.storage.tempDir=/path/to/your/temp/dir</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>For detailed information about the settings related to durable storage, see <a href="#durable-storage-configurations">Durable storage configurations</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="use-durable-storage-for-queries">Use durable storage for queries<a href="#use-durable-storage-for-queries" class="hash-link" aria-label="Direct link to Use durable storage for queries" title="Direct link to Use durable storage for queries"></a></h3><p>When you run a query, include the context parameter <code>durableShuffleStorage</code> and set it to <code>true</code>. </p><p>For queries where you want to use fault tolerance for workers, set <code>faultTolerance</code> to <code>true</code>, which automatically sets <code>durableShuffleStorage</code> to <code>true</code>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="durable-storage-configurations">Durable storage configurations<a href="#durable-storage-configurations" class="hash-link" aria-label="Direct link to Durable storage configurations" title="Direct link to Durable storage configurations"></a></h2><p>The following common service properties control how durable storage behaves:</p><table><thead><tr><th>Parameter</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td><code>druid.msq.intermediate.storage.bucket</code></td><td>n/a</td><td>The bucket in S3 where you want to store intermediate files.</td></tr><tr><td><code>druid.msq.intermediate.storage.chunkSize</code></td><td>100MiB</td><td>Optional. Defines the size of each chunk to temporarily store in <code>druid.msq.intermediate.storage.tempDir</code>. The chunk size must be between 5 MiB and 5 GiB. A large chunk size reduces the API calls made to the durable storage, however it requires more disk space to store the temporary chunks. Druid uses a default of 100MiB if the value is not provided.</td></tr><tr><td><code>druid.msq.intermediate.storage.enable</code></td><td>true</td><td>Required. Whether to enable durable storage for the cluster.</td></tr><tr><td><code>druid.msq.intermediate.storage.maxRetry</code></td><td>10</td><td>Optional. Defines the max number times to attempt S3 API calls to avoid failures due to transient errors.</td></tr><tr><td><code>druid.msq.intermediate.storage.prefix</code></td><td>n/a</td><td>S3 prefix to store intermediate stage results. Provide a unique value for the prefix. Don't share the same prefix between clusters. If the location includes other files or directories, then they will get cleaned up as well.</td></tr><tr><td><code>druid.msq.intermediate.storage.tempDir</code></td><td>n/a</td><td>Required. Directory path on the local disk to temporarily store intermediate stage results.</td></tr><tr><td><code>druid.msq.intermediate.storage.type</code></td><td><code>s3</code> if your deep storage is S3</td><td>Required. The type of storage to use. You can either set this to <code>local</code> or <code>s3</code>.</td></tr></tbody></table><p>In addition to the common service properties, there are certain properties that you configure on the Overlord specifically to clean up intermediate files:</p><table><thead><tr><th>Parameter</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td><code>druid.msq.intermediate.storage.cleaner.enabled</code></td><td>false</td><td>Optional. Whether durable storage cleaner should be enabled for the cluster.</td></tr><tr><td><code>druid.msq.intermediate.storage.cleaner.delaySeconds</code></td><td>86400</td><td>Optional. The delay (in seconds) after the last run post which the durable storage cleaner would clean the outputs.</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="limits">Limits<a href="#limits" class="hash-link" aria-label="Direct link to Limits" title="Direct link to Limits"></a></h2><p>Knowing the limits for the MSQ task engine can help you troubleshoot any <a href="#error-codes">errors</a> that you encounter. Many of the errors occur as a result of reaching a limit.</p><p>The following table lists query limits:</p><table><thead><tr><th>Limit</th><th>Value</th><th>Error if exceeded</th></tr></thead><tbody><tr><td>Size of an individual row written to a frame. Row size when written to a frame may differ from the original row size.</td><td>1 MB</td><td><a href="#error_RowTooLarge"><code>RowTooLarge</code></a></td></tr><tr><td>Number of segment-granular time chunks encountered during ingestion.</td><td>5,000</td><td><a href="#error_TooManyBuckets"><code>TooManyBuckets</code></a></td></tr><tr><td>Number of input files/segments per worker.</td><td>10,000</td><td><a href="#error_TooManyInputFiles"><code>TooManyInputFiles</code></a></td></tr><tr><td>Number of output partitions for any one stage. Number of segments generated during ingestion.</td><td>25,000</td><td><a href="#error_TooManyPartitions"><code>TooManyPartitions</code></a></td></tr><tr><td>Number of output columns for any one stage.</td><td>2,000</td><td><a href="#error_TooManyColumns"><code>TooManyColumns</code></a></td></tr><tr><td>Number of cluster by columns that can appear in a stage</td><td>1,500</td><td><a href="#error_TooManyClusteredByColumns"><code>TooManyClusteredByColumns</code></a></td></tr><tr><td>Number of workers for any one stage.</td><td>Hard limit is 1,000. Memory-dependent soft limit may be lower.</td><td><a href="#error_TooManyWorkers"><code>TooManyWorkers</code></a></td></tr><tr><td>Maximum memory occupied by broadcasted tables.</td><td>30% of each <a href="/docs/26.0.0/multi-stage-query/concepts#memory-usage">processor memory bundle</a>.</td><td><a href="#error_BroadcastTablesTooLarge"><code>BroadcastTablesTooLarge</code></a></td></tr><tr><td>Maximum memory occupied by buffered data during sort-merge join. Only relevant when <code>sqlJoinAlgorithm</code> is <code>sortMerge</code>.</td><td>10 MB</td><td><code>TooManyRowsWithSameKey</code></td></tr><tr><td>Maximum relaunch attempts per worker. Initial run is not a relaunch. The worker will be spawned 1 + <code>workerRelaunchLimit</code> times before the job fails.</td><td>2</td><td><code>TooManyAttemptsForWorker</code></td></tr><tr><td>Maximum relaunch attempts for a job across all workers.</td><td>100</td><td><code>TooManyAttemptsForJob</code></td></tr></tbody></table><a name="errors"></a><h2 class="anchor anchorWithStickyNavbar_LWe7" id="error-codes">Error codes<a href="#error-codes" class="hash-link" aria-label="Direct link to Error codes" title="Direct link to Error codes"></a></h2><p>The following table describes error codes you may encounter in the <code>multiStageQuery.payload.status.errorReport.error.errorCode</code> field:</p><table><thead><tr><th>Code</th><th>Meaning</th><th>Additional fields</th></tr></thead><tbody><tr><td><a name="error_BroadcastTablesTooLarge"><code>BroadcastTablesTooLarge</code></a></td><td>The size of the broadcast tables used in the right hand side of the join exceeded the memory reserved for them in a worker task.<br><br>Try increasing the peon memory or reducing the size of the broadcast tables.</td><td><code>maxBroadcastTablesSize</code>: Memory reserved for the broadcast tables, measured in bytes.</td></tr><tr><td><a name="error_Canceled"><code>Canceled</code></a></td><td>The query was canceled. Common reasons for cancellation:<br><br><ul><li>User-initiated shutdown of the controller task via the <code>/druid/indexer/v1/task/{taskId}/shutdown</code> API.</li><li>Restart or failure of the server process that was running the controller task.</li></ul></td><td></td></tr><tr><td><a name="error_CannotParseExternalData"><code>CannotParseExternalData</code></a></td><td>A worker task could not parse data from an external datasource.</td><td><code>errorMessage</code>: More details on why parsing failed.</td></tr><tr><td><a name="error_ColumnNameRestricted"><code>ColumnNameRestricted</code></a></td><td>The query uses a restricted column name.</td><td><code>columnName</code>: The restricted column name.</td></tr><tr><td><a name="error_ColumnTypeNotSupported"><code>ColumnTypeNotSupported</code></a></td><td>The column type is not supported. This can be because:<br> <br><ul><li>Support for writing or reading from a particular column type is not supported.</li><li>The query attempted to use a column type that is not supported by the frame format. This occurs with ARRAY types, which are not yet implemented for frames.</li></ul></td><td><code>columnName</code>: The column name with an unsupported type.<br> <br><code>columnType</code>: The unknown column type.</td></tr><tr><td><a name="error_InsertCannotAllocateSegment"><code>InsertCannotAllocateSegment</code></a></td><td>The controller task could not allocate a new segment ID due to conflict with existing segments or pending segments. Common reasons for such conflicts:<br> <br><ul><li>Attempting to mix different granularities in the same intervals of the same datasource.</li><li>Prior ingestions that used non-extendable shard specs.</li></ul></td><td><code>dataSource</code><br> <br><code>interval</code>: The interval for the attempted new segment allocation.</td></tr><tr><td><a name="error_InsertCannotBeEmpty"><code>InsertCannotBeEmpty</code></a></td><td>An INSERT or REPLACE query did not generate any output rows in a situation where output rows are required for success. This can happen for INSERT or REPLACE queries with <code>PARTITIONED BY</code> set to something other than <code>ALL</code> or <code>ALL TIME</code>.</td><td><code>dataSource</code></td></tr><tr><td><a name="error_InsertCannotOrderByDescending"><code>InsertCannotOrderByDescending</code></a></td><td>An INSERT query contained a <code>CLUSTERED BY</code> expression in descending order. Druid's segment generation code only supports ascending order.</td><td><code>columnName</code></td></tr><tr><td><a name="error_InsertLockPreempted"><code>InsertLockPreempted</code></a></td><td>An INSERT or REPLACE query was canceled by a higher-priority ingestion job, such as a real-time ingestion task.</td><td></td></tr><tr><td><a name="error_InsertTimeNull"><code>InsertTimeNull</code></a></td><td>An INSERT or REPLACE query encountered a null timestamp in the <code>__time</code> field.<br><br>This can happen due to using an expression like <code>TIME_PARSE(timestamp) AS __time</code> with a timestamp that cannot be parsed. (<a href="/docs/26.0.0/querying/sql-scalar#date-and-time-functions"><code>TIME_PARSE</code></a> returns null when it cannot parse a timestamp.) In this case, try parsing your timestamps using a different function or pattern. Or, if your timestamps may genuinely be null, consider using <a href="/docs/26.0.0/querying/sql-scalar#other-scalar-functions"><code>COALESCE</code></a> to provide a default value. One option is <a href="/docs/26.0.0/querying/sql-scalar#date-and-time-functions"><code>CURRENT_TIMESTAMP</code></a>, which represents the start time of the job.<br><br>This error code only occurs when <a href="/docs/26.0.0/configuration/#sql-compatible-null-handling"><code>druid.generic.useDefaultValueForNull = false</code></a>. When this property is <code>true</code> (the default setting), null timestamps are instead replaced by <code>1970-01-01 00:00:00</code>.</td><td></td></tr><tr><td><a name="error_InsertTimeOutOfBounds"><code>InsertTimeOutOfBounds</code></a></td><td>A REPLACE query generated a timestamp outside the bounds of the TIMESTAMP parameter for your OVERWRITE WHERE clause.<br> <br>To avoid this error, verify that the you specified is valid.</td><td><code>interval</code>: time chunk interval corresponding to the out-of-bounds timestamp</td></tr><tr><td><a name="error_InvalidNullByte"><code>InvalidNullByte</code></a></td><td>A string column included a null byte. Null bytes in strings are not permitted.</td><td><code>column</code>: The column that included the null byte</td></tr><tr><td><a name="error_QueryNotSupported"><code>QueryNotSupported</code></a></td><td>QueryKit could not translate the provided native query to a multi-stage query.<br> <br>This can happen if the query uses features that aren't supported, like GROUPING SETS.</td><td></td></tr><tr><td><a name="error_QueryRuntimeError"><code>QueryRuntimeError</code></a></td><td>MSQ uses the native query engine to run the leaf stages. This error tells MSQ that error is in native query runtime.<br> <br> Since this is a generic error, the user needs to look at logs for the error message and stack trace to figure out the next course of action. If the user is stuck, consider raising a <code>github</code> issue for assistance.</td><td><code>baseErrorMessage</code> error message from the native query runtime.</td></tr><tr><td><a name="error_RowTooLarge"><code>RowTooLarge</code></a></td><td>The query tried to process a row that was too large to write to a single frame. See the <a href="#limits">Limits</a> table for specific limits on frame size. Note that the effective maximum row size is smaller than the maximum frame size due to alignment considerations during frame writing.</td><td><code>maxFrameSize</code>: The limit on the frame size.</td></tr><tr><td><a name="error_TaskStartTimeout"><code>TaskStartTimeout</code></a></td><td>Unable to launch <code>numTasks</code> tasks within <code>timeout</code> milliseconds.<br><br>There may be insufficient available slots to start all the worker tasks simultaneously. Try splitting up your query into smaller chunks using a smaller value of <a href="#context-parameters"><code>maxNumTasks</code></a>. Another option is to increase capacity.</td><td><code>numTasks</code>: The number of tasks attempted to launch.<br><br><code>timeout</code>: Timeout, in milliseconds, that was exceeded.</td></tr><tr><td><a name="error_TooManyAttemptsForJob"><code>TooManyAttemptsForJob</code></a></td><td>Total relaunch attempt count across all workers exceeded max relaunch attempt limit. See the <a href="#limits">Limits</a> table for the specific limit.</td><td><code>maxRelaunchCount</code>: Max number of relaunches across all the workers defined in the <a href="#limits">Limits</a> section. <br><br> <code>currentRelaunchCount</code>: current relaunch counter for the job across all workers. <br><br> <code>taskId</code>: Latest task id which failed <br> <br> <code>rootErrorMessage</code>: Error message of the latest failed task.</td></tr><tr><td><a name="error_TooManyAttemptsForWorker"><code>TooManyAttemptsForWorker</code></a></td><td>Worker exceeded maximum relaunch attempt count as defined in the <a href="#limits">Limits</a> section.</td><td><code>maxPerWorkerRelaunchCount</code>: Max number of relaunches allowed per worker as defined in the <a href="#limits">Limits</a> section. <br><br> <code>workerNumber</code>: the worker number for which the task failed <br><br> <code>taskId</code>: Latest task id which failed <br> <br> <code>rootErrorMessage</code>: Error message of the latest failed task.</td></tr><tr><td><a name="error_TooManyBuckets"><code>TooManyBuckets</code></a></td><td>Exceeded the maximum number of partition buckets for a stage (5,000 partition buckets).<br>< br />Partition buckets are created for each <a href="#partitioned-by"><code>PARTITIONED BY</code></a> time chunk for INSERT and REPLACE queries. The most common reason for this error is that your <code>PARTITIONED BY</code> is too narrow relative to your data.</td><td><code>maxBuckets</code>: The limit on partition buckets.</td></tr><tr><td><a name="error_TooManyInputFiles"><code>TooManyInputFiles</code></a></td><td>Exceeded the maximum number of input files or segments per worker (10,000 files or segments).<br><br>If you encounter this limit, consider adding more workers, or breaking up your query into smaller queries that process fewer files or segments per query.</td><td><code>numInputFiles</code>: The total number of input files/segments for the stage.<br><br><code>maxInputFiles</code>: The maximum number of input files/segments per worker per stage.<br><br><code>minNumWorker</code>: The minimum number of workers required for a successful run.</td></tr><tr><td><a name="error_TooManyPartitions"><code>TooManyPartitions</code></a></td><td>Exceeded the maximum number of partitions for a stage (25,000 partitions).<br><br>This can occur with INSERT or REPLACE statements that generate large numbers of segments, since each segment is associated with a partition. If you encounter this limit, consider breaking up your INSERT or REPLACE statement into smaller statements that process less data per statement.</td><td><code>maxPartitions</code>: The limit on partitions which was exceeded</td></tr><tr><td><a name="error_TooManyClusteredByColumns"><code>TooManyClusteredByColumns</code></a></td><td>Exceeded the maximum number of clustering columns for a stage (1,500 columns).<br><br>This can occur with <code>CLUSTERED BY</code>, <code>ORDER BY</code>, or <code>GROUP BY</code> with a large number of columns.</td><td><code>numColumns</code>: The number of columns requested.<br><br><code>maxColumns</code>: The limit on columns which was exceeded.<code>stage</code>: The stage number exceeding the limit<br><br></td></tr><tr><td><a name="error_TooManyRowsWithSameKey"><code>TooManyRowsWithSameKey</code></a></td><td>The number of rows for a given key exceeded the maximum number of buffered bytes on both sides of a join. See the <a href="#limits">Limits</a> table for the specific limit. Only occurs when <code>sqlJoinAlgorithm</code> is <code>sortMerge</code>.</td><td><code>key</code>: The key that had a large number of rows.<br><br><code>numBytes</code>: Number of bytes buffered, which may include other keys.<br><br><code>maxBytes</code>: Maximum number of bytes buffered.</td></tr><tr><td><a name="error_TooManyColumns"><code>TooManyColumns</code></a></td><td>Exceeded the maximum number of columns for a stage (2,000 columns).</td><td><code>numColumns</code>: The number of columns requested.<br><br><code>maxColumns</code>: The limit on columns which was exceeded.</td></tr><tr><td><a name="error_TooManyWarnings"><code>TooManyWarnings</code></a></td><td>Exceeded the maximum allowed number of warnings of a particular type.</td><td><code>rootErrorCode</code>: The error code corresponding to the exception that exceeded the required limit. <br><br><code>maxWarnings</code>: Maximum number of warnings that are allowed for the corresponding <code>rootErrorCode</code>.</td></tr><tr><td><a name="error_TooManyWorkers"><code>TooManyWorkers</code></a></td><td>Exceeded the maximum number of simultaneously-running workers. See the <a href="#limits">Limits</a> table for more details.</td><td><code>workers</code>: The number of simultaneously running workers that exceeded a hard or soft limit. This may be larger than the number of workers in any one stage if multiple stages are running simultaneously. <br><br><code>maxWorkers</code>: The hard or soft limit on workers that was exceeded. If this is lower than the hard limit (1,000 workers), then you can increase the limit by adding more memory to each task.</td></tr><tr><td><a name="error_NotEnoughMemory"><code>NotEnoughMemory</code></a></td><td>Insufficient memory to launch a stage.</td><td><code>suggestedServerMemory</code>: Suggested number of bytes of memory to allocate to a given process. <br><br><code>serverMemory</code>: The number of bytes of memory available to a single process.<br><br><code>usableMemory</code>: The number of usable bytes of memory for a single process.<br><br><code>serverWorkers</code>: The number of workers running in a single process.<br><br><code>serverThreads</code>: The number of threads in a single process.</td></tr><tr><td><a name="error_NotEnoughTemporaryStorage"><code>NotEnoughTemporaryStorage</code></a></td><td>Insufficient temporary storage configured to launch a stage. This limit is set by the property <code>druid.indexer.task.tmpStorageBytesPerTask</code>. This property should be increased to the minimum suggested limit to resolve this.</td><td><code>suggestedMinimumStorage</code>: Suggested number of bytes of temporary storage space to allocate to a given process. <br><br><code>configuredTemporaryStorage</code>: The number of bytes of storage currently configured.</td></tr><tr><td><a name="error_WorkerFailed"><code>WorkerFailed</code></a></td><td>A worker task failed unexpectedly.</td><td><code>errorMsg</code><br><br><code>workerTaskId</code>: The ID of the worker task.</td></tr><tr><td><a name="error_WorkerRpcFailed"><code>WorkerRpcFailed</code></a></td><td>A remote procedure call to a worker task failed and could not recover.</td><td><code>workerTaskId</code>: the id of the worker task</td></tr><tr><td><a name="error_UnknownError"><code>UnknownError</code></a></td><td>All other errors.</td><td><code>message</code></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/docs/26.0.0/multi-stage-query/examples"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Examples</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/docs/26.0.0/multi-stage-query/known-issues"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Known issues</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#sql-reference" class="table-of-contents__link toc-highlight">SQL reference</a><ul><li><a href="#extern-function" class="table-of-contents__link toc-highlight"><code>EXTERN</code> Function</a></li><li><a href="#insert" class="table-of-contents__link toc-highlight"><code>INSERT</code></a></li><li><a href="#replace" class="table-of-contents__link toc-highlight"><code>REPLACE</code></a></li><li><a href="#partitioned-by" class="table-of-contents__link toc-highlight"><code>PARTITIONED BY</code></a></li><li><a href="#clustered-by" class="table-of-contents__link toc-highlight"><code>CLUSTERED BY</code></a></li></ul></li><li><a href="#context-parameters" class="table-of-contents__link toc-highlight">Context parameters</a></li><li><a href="#joins" class="table-of-contents__link toc-highlight">Joins</a><ul><li><a href="#broadcast" class="table-of-contents__link toc-highlight">Broadcast</a></li><li><a href="#sort-merge" class="table-of-contents__link toc-highlight">Sort-merge</a></li></ul></li><li><a href="#durable-storage" class="table-of-contents__link toc-highlight">Durable Storage</a><ul><li><a href="#enable-durable-storage" class="table-of-contents__link toc-highlight">Enable durable storage</a></li><li><a href="#use-durable-storage-for-queries" class="table-of-contents__link toc-highlight">Use durable storage for queries</a></li></ul></li><li><a href="#durable-storage-configurations" class="table-of-contents__link toc-highlight">Durable storage configurations</a></li><li><a href="#limits" class="table-of-contents__link toc-highlight">Limits</a></li><li><a href="#error-codes" class="table-of-contents__link toc-highlight">Error codes</a></li></ul></div></div></div></div></main></div></div><footer class="footer"><div class="container container-fluid"><div class="footer__bottom text--center"><div class="margin-bottom--sm"><img src="/img/favicon.png" class="themedImage_ToTc themedImage--light_HNdA footer__logo"><img src="/img/favicon.png" class="themedImage_ToTc themedImage--dark_i4oU footer__logo"></div><div class="footer__copyright">Copyright © 2023 Apache Software Foundation. Except where otherwise noted, licensed under CC BY-SA 4.0. Apache Druid, Druid, and the Druid logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</div></div></div></footer></div> |
| <script src="/assets/js/runtime~main.38900cbf.js"></script> |
| <script src="/assets/js/main.5e106d68.js"></script> |
| </body> |
| </html> |
