| <!doctype html> |
| <html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-current plugin-docs plugin-id-default docs-doc-id-ingestion/ingestion-spec"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="generator" content="Docusaurus v2.4.1"> |
| <title data-rh="true">Ingestion spec reference | Apache® Druid</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:image" content="https://druid.apache.org/img/druid_nav.png"><meta data-rh="true" name="twitter:image" content="https://druid.apache.org/img/druid_nav.png"><meta data-rh="true" property="og:url" content="https://druid.apache.org/docs/28.0.0/ingestion/ingestion-spec"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="current"><meta data-rh="true" name="docusaurus_tag" content="docs-default-current"><meta data-rh="true" name="docsearch:version" content="current"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-current"><meta data-rh="true" property="og:title" content="Ingestion spec reference | Apache® Druid"><meta data-rh="true" name="description" content="Reference for the configuration options in the ingestion spec."><meta data-rh="true" property="og:description" content="Reference for the configuration options in the ingestion spec."><link data-rh="true" rel="icon" href="/img/favicon.png"><link data-rh="true" rel="canonical" href="https://druid.apache.org/docs/28.0.0/ingestion/ingestion-spec"><link data-rh="true" rel="alternate" href="https://druid.apache.org/docs/28.0.0/ingestion/ingestion-spec" hreflang="en"><link data-rh="true" rel="alternate" href="https://druid.apache.org/docs/28.0.0/ingestion/ingestion-spec" hreflang="x-default"><link rel="preconnect" href="https://www.google-analytics.com"> |
| <link rel="preconnect" href="https://www.googletagmanager.com"> |
| <script async src="https://www.googletagmanager.com/gtag/js?id=UA-131010415-1"></script> |
| <script>function gtag(){dataLayer.push(arguments)}window.dataLayer=window.dataLayer||[],gtag("js",new Date),gtag("config","UA-131010415-1",{})</script> |
| |
| |
| |
| |
| <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"> |
| <script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><link rel="stylesheet" href="/assets/css/styles.546f39eb.css"> |
| <link rel="preload" href="/assets/js/runtime~main.0dcbfdea.js" as="script"> |
| <link rel="preload" href="/assets/js/main.7f6fdf81.js" as="script"> |
| </head> |
| <body class="navigation-with-keyboard"> |
| <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus"> |
| <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><nav aria-label="Main" class="navbar navbar--fixed-top navbar--dark"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/druid_nav.png" alt="Apache® Druid" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/druid_nav.png" alt="Apache® Druid" class="themedImage_ToTc themedImage--dark_i4oU"></div></a></div><div class="navbar__items navbar__items--right"><a class="navbar__item navbar__link" href="/technology">Technology</a><a class="navbar__item navbar__link" href="/use-cases">Use Cases</a><a class="navbar__item navbar__link" href="/druid-powered">Powered By</a><a class="navbar__item navbar__link" href="/docs/28.0.0/design/">Docs</a><a class="navbar__item navbar__link" href="/community/">Community</a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Apache®</a><ul class="dropdown__menu"><li><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Foundation<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://apachecon.com/?ref=druid.apache.org" target="_blank" rel="noopener noreferrer" class="dropdown__link">Events<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/licenses/" target="_blank" rel="noopener noreferrer" class="dropdown__link">License<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Thanks<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/security/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Security<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Sponsorship<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a class="navbar__item navbar__link" href="/downloads/">Download</a><div class="searchBox_ZlJk"><div class="navbar__search"><span aria-label="expand searchbar" role="button" class="search-icon" tabindex="0"></span><input type="search" id="search_input_react" placeholder="Loading..." aria-label="Search" class="navbar__search-input search-bar" disabled=""></div></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.0/design/">Getting started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.0/tutorials/tutorial-msq-extern">Tutorials</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.0/design/architecture">Design</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/docs/28.0.0/ingestion/">Ingestion</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/28.0.0/ingestion/">Overview</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/28.0.0/ingestion/data-formats">Ingestion concepts</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/28.0.0/multi-stage-query/">SQL-based batch</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/28.0.0/development/extensions-core/kafka-ingestion">Streaming</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/28.0.0/ingestion/native-batch">Classic batch</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/docs/28.0.0/ingestion/ingestion-spec">Ingestion spec reference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/28.0.0/ingestion/schema-design">Schema design tips</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/28.0.0/ingestion/faq">Troubleshooting FAQ</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.0/data-management/">Data management</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.0/querying/sql">Querying</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.0/api-reference/">API reference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.0/configuration/">Configuration</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.0/operations/web-console">Operations</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.0/development/overview">Development</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/28.0.0/misc/papers-and-talks">Misc</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Ingestion</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Ingestion spec reference</span><meta itemprop="position" content="2"></li></ul></nav><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Ingestion spec reference</h1></header><p>All ingestion methods use ingestion tasks to load data into Druid. Streaming ingestion uses ongoing supervisors that run and supervise a set of tasks over time. Native batch and Hadoop-based ingestion use a one-time <a href="/docs/28.0.0/ingestion/tasks">task</a>. Other than with SQL-based ingestion, use an <em>ingestion spec</em> to configure your ingestion.</p><p>Ingestion specs consists of three main components:</p><ul><li><a href="#dataschema"><code>dataSchema</code></a>, which configures the <a href="#datasource">datasource name</a>, |
| <a href="#timestampspec">primary timestamp</a>, <a href="#dimensionsspec">dimensions</a>, <a href="#metricsspec">metrics</a>, and <a href="#transformspec">transforms and filters</a> (if needed).</li><li><a href="#ioconfig"><code>ioConfig</code></a>, which tells Druid how to connect to the source system and how to parse data. For more information, see the |
| documentation for each <a href="/docs/28.0.0/ingestion/#ingestion-methods">ingestion method</a>.</li><li><a href="#tuningconfig"><code>tuningConfig</code></a>, which controls various tuning parameters specific to each |
| <a href="/docs/28.0.0/ingestion/#ingestion-methods">ingestion method</a>.</li></ul><p>Example ingestion spec for task type <code>index_parallel</code> (native batch):</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "type": "index_parallel",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "spec": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "dataSchema": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "dataSource": "wikipedia",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "timestampSpec": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "column": "timestamp",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "format": "auto"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "dimensionsSpec": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "dimensions": [</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "page",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "language",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> { "type": "long", "name": "userId" }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> ]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "metricsSpec": [</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> { "type": "count", "name": "count" },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> { "type": "doubleSum", "name": "bytes_added_sum", "fieldName": "bytes_added" },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> { "type": "doubleSum", "name": "bytes_deleted_sum", "fieldName": "bytes_deleted" }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> ],</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "granularitySpec": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "segmentGranularity": "day",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "queryGranularity": "none",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "intervals": [</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "2013-08-31/2013-09-01"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> ]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "ioConfig": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "type": "index_parallel",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "inputSource": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "type": "local",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "baseDir": "examples/indexing/",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "filter": "wikipedia_data.json"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "inputFormat": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "type": "json",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "flattenSpec": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "useFieldDiscovery": true,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "fields": [</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> { "type": "path", "name": "userId", "expr": "$.user.id" }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> ]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "tuningConfig": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "type": "index_parallel"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The specific options supported by these sections will depend on the <a href="/docs/28.0.0/ingestion/#ingestion-methods">ingestion method</a> you have chosen. |
| For more examples, refer to the documentation for each ingestion method.</p><p>You can also load data visually, without the need to write an ingestion spec, using the "Load data" functionality |
| available in Druid's <a href="/docs/28.0.0/operations/web-console">web console</a>. Druid's visual data loader supports |
| <a href="/docs/28.0.0/development/extensions-core/kafka-ingestion">Kafka</a>, |
| <a href="/docs/28.0.0/development/extensions-core/kinesis-ingestion">Kinesis</a>, and |
| <a href="/docs/28.0.0/ingestion/native-batch">native batch</a> mode.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="dataschema"><code>dataSchema</code><a href="#dataschema" class="hash-link" aria-label="Direct link to dataschema" title="Direct link to dataschema"></a></h2><div class="theme-admonition theme-admonition-info alert alert--info admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg></span>info</div><div class="admonitionContent_S0QG"><p> The <code>dataSchema</code> spec has been changed in 0.17.0. The new spec is supported by all ingestion methods |
| except for <em>Hadoop</em> ingestion. See the <a href="#legacy-dataschema-spec">Legacy <code>dataSchema</code> spec</a> for the old spec.</p></div></div><p>The <code>dataSchema</code> is a holder for the following components:</p><ul><li><a href="#datasource">datasource name</a></li><li><a href="#timestampspec">primary timestamp</a></li><li><a href="#dimensionsspec">dimensions</a></li><li><a href="#metricsspec">metrics</a></li><li><a href="#transformspec">transforms and filters</a> (if needed).</li></ul><p>An example <code>dataSchema</code> is:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">"dataSchema": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "dataSource": "wikipedia",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "timestampSpec": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "column": "timestamp",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "format": "auto"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "dimensionsSpec": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "dimensions": [</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "page",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "language",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> { "type": "long", "name": "userId" }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> ]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "metricsSpec": [</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> { "type": "count", "name": "count" },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> { "type": "doubleSum", "name": "bytes_added_sum", "fieldName": "bytes_added" },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> { "type": "doubleSum", "name": "bytes_deleted_sum", "fieldName": "bytes_deleted" }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> ],</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "granularitySpec": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "segmentGranularity": "day",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "queryGranularity": "none",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "intervals": [</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "2013-08-31/2013-09-01"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> ]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="datasource"><code>dataSource</code><a href="#datasource" class="hash-link" aria-label="Direct link to datasource" title="Direct link to datasource"></a></h3><p>The <code>dataSource</code> is located in <code>dataSchema</code> → <code>dataSource</code> and is simply the name of the |
| <a href="/docs/28.0.0/design/architecture#datasources-and-segments">datasource</a> that data will be written to. An example |
| <code>dataSource</code> is:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">"dataSource": "my-first-datasource"</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="timestampspec"><code>timestampSpec</code><a href="#timestampspec" class="hash-link" aria-label="Direct link to timestampspec" title="Direct link to timestampspec"></a></h3><p>The <code>timestampSpec</code> is located in <code>dataSchema</code> → <code>timestampSpec</code> and is responsible for |
| configuring the <a href="/docs/28.0.0/ingestion/schema-model#primary-timestamp">primary timestamp</a>. An example <code>timestampSpec</code> is:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">"timestampSpec": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "column": "timestamp",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "format": "auto"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="theme-admonition theme-admonition-info alert alert--info admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg></span>info</div><div class="admonitionContent_S0QG"><p> Conceptually, after input data records are read, Druid applies ingestion spec components in a particular order: |
| first <a href="/docs/28.0.0/ingestion/data-formats#flattenspec"><code>flattenSpec</code></a> (if any), then <a href="#timestampspec"><code>timestampSpec</code></a>, then <a href="#transformspec"><code>transformSpec</code></a>, |
| and finally <a href="#dimensionsspec"><code>dimensionsSpec</code></a> and <a href="#metricsspec"><code>metricsSpec</code></a>. Keep this in mind when writing |
| your ingestion spec.</p></div></div><p>A <code>timestampSpec</code> can have the following components:</p><table><thead><tr><th>Field</th><th>Description</th><th>Default</th></tr></thead><tbody><tr><td>column</td><td>Input row field to read the primary timestamp from.<br><br>Regardless of the name of this input field, the primary timestamp will always be stored as a column named <code>__time</code> in your Druid datasource.</td><td>timestamp</td></tr><tr><td>format</td><td>Timestamp format. Options are: <ul><li><code>iso</code>: ISO8601 with 'T' separator, like "2000-01-01T01:02:03.456"</li><li><code>posix</code>: seconds since epoch</li><li><code>millis</code>: milliseconds since epoch</li><li><code>micro</code>: microseconds since epoch</li><li><code>nano</code>: nanoseconds since epoch</li><li><code>auto</code>: automatically detects ISO (either 'T' or space separator) or millis format</li><li>any <a href="http://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html" target="_blank" rel="noopener noreferrer">Joda DateTimeFormat string</a></li></ul></td><td>auto</td></tr><tr><td>missingValue</td><td>Timestamp to use for input records that have a null or missing timestamp <code>column</code>. Should be in ISO8601 format, like <code>"2000-01-01T01:02:03.456"</code>, even if you have specified something else for <code>format</code>. Since Druid requires a primary timestamp, this setting can be useful for ingesting datasets that do not have any per-record timestamps at all.</td><td>none</td></tr></tbody></table><p>You can use the timestamp in a expression as <code>__time</code> because Druid parses the <code>timestampSpec</code> before applying <a href="#transforms">transforms</a>. You can also set the expression <code>name</code> to <code>__time</code> to replace the value of the timestamp.</p><p>Treat <code>__time</code> as a millisecond timestamp: the number of milliseconds since Jan 1, 1970 at midnight UTC.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dimensionsspec"><code>dimensionsSpec</code><a href="#dimensionsspec" class="hash-link" aria-label="Direct link to dimensionsspec" title="Direct link to dimensionsspec"></a></h3><p>The <code>dimensionsSpec</code> is located in <code>dataSchema</code> → <code>dimensionsSpec</code> and is responsible for |
| configuring <a href="/docs/28.0.0/ingestion/schema-model#dimensions">dimensions</a>.</p><p>You can either manually specify the dimensions or take advantage of schema auto-discovery where you allow Druid to infer all or some of the schema for your data. This means that you don't have to explicitly specify your dimensions and their type. </p><p>To use schema auto-discovery, set <code>useSchemaDiscovery</code> to <code>true</code>. </p><p>Alternatively, you can use the string-based schemaless ingestion where any discovered dimensions are treated as strings. To do so, leave <code>useSchemaDiscovery</code> set to <code>false</code> (default). Then, set the dimensions list to empty or set the <code>includeAllDimensions</code> property to <code>true</code>.</p><p>The following <code>dimensionsSpec</code> example uses schema auto-discovery (<code>"useSchemaDiscovery": true</code>) in conjunction with explicitly defined dimensions to have Druid infer some of the schema for the data:</p><div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token property">"dimensionsSpec"</span><span class="token plain"> </span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"dimensions"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"page"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"language"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"long"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token property">"name"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"userId"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"dimensionExclusions"</span><span class="token plain"> </span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"spatialDimensions"</span><span class="token plain"> </span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"useSchemaDiscovery"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token boolean" style="color:rgb(255, 88, 116)">true</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="theme-admonition theme-admonition-info alert alert--info admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg></span>info</div><div class="admonitionContent_S0QG"><p> Conceptually, after input data records are read, Druid applies ingestion spec components in a particular order: |
| first <a href="/docs/28.0.0/ingestion/data-formats#flattenspec"><code>flattenSpec</code></a> (if any), then <a href="#timestampspec"><code>timestampSpec</code></a>, then <a href="#transformspec"><code>transformSpec</code></a>, |
| and finally <a href="#dimensionsspec"><code>dimensionsSpec</code></a> and <a href="#metricsspec"><code>metricsSpec</code></a>. Keep this in mind when writing |
| your ingestion spec.</p></div></div><p>A <code>dimensionsSpec</code> can have the following components:</p><table><thead><tr><th>Field</th><th>Description</th><th>Default</th></tr></thead><tbody><tr><td><code>dimensions</code></td><td>A list of <a href="#dimension-objects">dimension names or objects</a>. You cannot include the same column in both <code>dimensions</code> and <code>dimensionExclusions</code>.<br><br>If <code>dimensions</code> and <code>spatialDimensions</code> are both null or empty arrays, Druid treats all columns other than timestamp or metrics that do not appear in <code>dimensionExclusions</code> as String-typed dimension columns. See <a href="#inclusions-and-exclusions">inclusions and exclusions</a> for details.<br><br>As a best practice, put the most frequently filtered dimensions at the beginning of the dimensions list. In this case, it would also be good to consider <a href="/docs/28.0.0/ingestion/partitioning"><code>partitioning</code></a> by those same dimensions.</td><td><code>[]</code></td></tr><tr><td><code>dimensionExclusions</code></td><td>The names of dimensions to exclude from ingestion. Only names are supported here, not objects.<br><br>This list is only used if the <code>dimensions</code> and <code>spatialDimensions</code> lists are both null or empty arrays; otherwise it is ignored. See <a href="#inclusions-and-exclusions">inclusions and exclusions</a> below for details.</td><td><code>[]</code></td></tr><tr><td><code>spatialDimensions</code></td><td>An array of <a href="/docs/28.0.0/querying/geo">spatial dimensions</a>.</td><td><code>[]</code></td></tr><tr><td><code>includeAllDimensions</code></td><td>Note that this field only applies to string-based schema discovery where Druid ingests dimensions it discovers as strings. This is different from schema auto-discovery where Druid infers the type for data. You can set <code>includeAllDimensions</code> to true to ingest both explicit dimensions in the <code>dimensions</code> field and other dimensions that the ingestion task discovers from input data. In this case, the explicit dimensions will appear first in the order that you specify them, and the dimensions dynamically discovered will come after. This flag can be useful especially with auto schema discovery using <a href="/docs/28.0.0/ingestion/data-formats#flattenspec"><code>flattenSpec</code></a>. If this is not set and the <code>dimensions</code> field is not empty, Druid will ingest only explicit dimensions. If this is not set and the <code>dimensions</code> field is empty, all discovered dimensions will be ingested.</td><td>false</td></tr><tr><td><code>useSchemaDiscovery</code></td><td>Configure Druid to use schema auto-discovery to discover some or all of the dimensions and types for your data. For any dimensions that aren't a uniform type, Druid ingests them as JSON. You can use this for native batch or streaming ingestion.</td><td>false</td></tr></tbody></table><h4 class="anchor anchorWithStickyNavbar_LWe7" id="dimension-objects">Dimension objects<a href="#dimension-objects" class="hash-link" aria-label="Direct link to Dimension objects" title="Direct link to Dimension objects"></a></h4><p>Each dimension in the <code>dimensions</code> list can either be a name or an object. Providing a name is equivalent to providing |
| a <code>string</code> type dimension object with the given name, e.g. <code>"page"</code> is equivalent to <code>{"name": "page", "type": "string"}</code>.</p><p>Dimension objects can have the following components:</p><table><thead><tr><th>Field</th><th>Description</th><th>Default</th></tr></thead><tbody><tr><td>type</td><td>Either <code>auto</code>, <code>string</code>, <code>long</code>, <code>float</code>, <code>double</code>, or <code>json</code>. For the <code>auto</code> type, Druid determines the most appropriate type for the dimension and assigns one of the following: STRING, ARRAY<STRING<!-- -->><!-- -->, LONG, ARRAY<LONG<!-- -->><!-- -->, DOUBLE, ARRAY<DOUBLE<!-- -->><!-- -->, or COMPLEX<json<!-- -->><!-- --> columns, all sharing a common 'nested' format. When Druid infers the schema with schema auto-discovery, the type is <code>auto</code>.</td><td><code>string</code></td></tr><tr><td>name</td><td>The name of the dimension. This will be used as the field name to read from input records, as well as the column name stored in generated segments.<br><br>Note that you can use a <a href="#transformspec"><code>transformSpec</code></a> if you want to rename columns during ingestion time.</td><td>none (required)</td></tr><tr><td>createBitmapIndex</td><td>For <code>string</code> typed dimensions, whether or not bitmap indexes should be created for the column in generated segments. Creating a bitmap index requires more storage, but speeds up certain kinds of filtering (especially equality and prefix filtering). Only supported for <code>string</code> typed dimensions.</td><td><code>true</code></td></tr><tr><td>multiValueHandling</td><td>For <code>string</code> typed dimensions, specifies the type of handling for <a href="/docs/28.0.0/querying/multi-value-dimensions">multi-value fields</a>. Possible values are <code>array</code> (ingest string arrays as-is), <code>sorted_array</code> (sort string arrays during ingestion), and <code>sorted_set</code> (sort and de-duplicate string arrays during ingestion). This parameter is ignored for types other than <code>string</code>.</td><td><code>sorted_array</code></td></tr></tbody></table><h4 class="anchor anchorWithStickyNavbar_LWe7" id="inclusions-and-exclusions">Inclusions and exclusions<a href="#inclusions-and-exclusions" class="hash-link" aria-label="Direct link to Inclusions and exclusions" title="Direct link to Inclusions and exclusions"></a></h4><p>Druid will interpret a <code>dimensionsSpec</code> in two possible ways: <em>normal</em> or <em>schemaless</em>.</p><p>Normal interpretation occurs when either <code>dimensions</code> or <code>spatialDimensions</code> is non-empty. In this case, the combination of the two lists will be taken as the set of dimensions to be ingested, and the list of <code>dimensionExclusions</code> will be ignored.</p><div class="theme-admonition theme-admonition-info alert alert--info admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg></span>info</div><div class="admonitionContent_S0QG"><p> The following description of schemaless refers to string-based schemaless where Druid treats dimensions it discovers as strings. We recommend you use schema auto-discovery instead where Druid infers the type for the dimension. For more information, see <a href="#dimensionsspec"><code>dimensionsSpec</code></a>.</p></div></div><p>Schemaless interpretation occurs when both <code>dimensions</code> and <code>spatialDimensions</code> are empty or null. In this case, the set of dimensions is determined in the following way:</p><ol><li>First, start from the set of all root-level fields from the input record, as determined by the <a href="/docs/28.0.0/ingestion/data-formats"><code>inputFormat</code></a>. "Root-level" includes all fields at the top level of a data structure, but does not included fields nested within maps or lists. To extract these, you must use a <a href="/docs/28.0.0/ingestion/data-formats#flattenspec"><code>flattenSpec</code></a>. All fields of non-nested data formats, such as CSV and delimited text, are considered root-level.</li><li>If a <a href="/docs/28.0.0/ingestion/data-formats#flattenspec"><code>flattenSpec</code></a> is being used, the set of root-level fields includes any fields generated by the <code>flattenSpec</code>. The <code>useFieldDiscovery</code> parameter determines whether the original root-level fields will be retained or discarded.</li><li>Any field listed in <code>dimensionExclusions</code> is excluded.</li><li>The field listed as <code>column</code> in the <a href="#timestampspec"><code>timestampSpec</code></a> is excluded.</li><li>Any field used as an input to an aggregator from the <a href="#metricsspec">metricsSpec</a> is excluded.</li><li>Any field with the same name as an aggregator from the <a href="#metricsspec">metricsSpec</a> is excluded.</li><li>All other fields are ingested as <code>string</code> typed dimensions with the <a href="#dimension-objects">default settings</a>.</li></ol><p>Additionally, if you have empty columns that you want to include in the string-based schemaless ingestion, you'll need to include the context parameter <code>storeEmptyColumns</code> and set it to <code>true</code>.</p><div class="theme-admonition theme-admonition-info alert alert--info admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg></span>info</div><div class="admonitionContent_S0QG"><p> Note: Fields generated by a <a href="#transformspec"><code>transformSpec</code></a> are not currently considered candidates for |
| schemaless dimension interpretation.</p></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="metricsspec"><code>metricsSpec</code><a href="#metricsspec" class="hash-link" aria-label="Direct link to metricsspec" title="Direct link to metricsspec"></a></h3><p>The <code>metricsSpec</code> is located in <code>dataSchema</code> → <code>metricsSpec</code> and is a list of <a href="/docs/28.0.0/querying/aggregations">aggregators</a> |
| to apply at ingestion time. This is most useful when <a href="/docs/28.0.0/ingestion/rollup">rollup</a> is enabled, since it's how you configure |
| ingestion-time aggregation.</p><p>An example <code>metricsSpec</code> is:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">"metricsSpec": [</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> { "type": "count", "name": "count" },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> { "type": "doubleSum", "name": "bytes_added_sum", "fieldName": "bytes_added" },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> { "type": "doubleSum", "name": "bytes_deleted_sum", "fieldName": "bytes_deleted" }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="theme-admonition theme-admonition-info alert alert--info admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg></span>info</div><div class="admonitionContent_S0QG"><p> Generally, when <a href="/docs/28.0.0/ingestion/rollup">rollup</a> is disabled, you should have an empty <code>metricsSpec</code> (because without rollup, |
| Druid does not do any ingestion-time aggregation, so there is little reason to include an ingestion-time aggregator). However, |
| in some cases, it can still make sense to define metrics: for example, if you want to create a complex column as a way of |
| pre-computing part of an <a href="/docs/28.0.0/querying/aggregations#approximate-aggregations">approximate aggregation</a>, this can only |
| be done by defining a metric in a <code>metricsSpec</code>.</p></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="granularityspec"><code>granularitySpec</code><a href="#granularityspec" class="hash-link" aria-label="Direct link to granularityspec" title="Direct link to granularityspec"></a></h3><p>The <code>granularitySpec</code> is located in <code>dataSchema</code> → <code>granularitySpec</code> and is responsible for configuring |
| the following operations:</p><ol><li>Partitioning a datasource into <a href="/docs/28.0.0/design/architecture#datasources-and-segments">time chunks</a> (via <code>segmentGranularity</code>).</li><li>Truncating the timestamp, if desired (via <code>queryGranularity</code>).</li><li>Specifying which time chunks of segments should be created, for batch ingestion (via <code>intervals</code>).</li><li>Specifying whether ingestion-time <a href="/docs/28.0.0/ingestion/rollup">rollup</a> should be used or not (via <code>rollup</code>).</li></ol><p>Other than <code>rollup</code>, these operations are all based on the <a href="/docs/28.0.0/ingestion/schema-model#primary-timestamp">primary timestamp</a>.</p><p>An example <code>granularitySpec</code> is:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">"granularitySpec": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "segmentGranularity": "day",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "queryGranularity": "none",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "intervals": [</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "2013-08-31/2013-09-01"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> ],</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "rollup": true</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>A <code>granularitySpec</code> can have the following components:</p><table><thead><tr><th>Field</th><th>Description</th><th>Default</th></tr></thead><tbody><tr><td>type</td><td><code>uniform</code></td><td><code>uniform</code></td></tr><tr><td>segmentGranularity</td><td><a href="/docs/28.0.0/design/architecture#datasources-and-segments">Time chunking</a> granularity for this datasource. Multiple segments can be created per time chunk. For example, when set to <code>day</code>, the events of the same day fall into the same time chunk which can be optionally further partitioned into multiple segments based on other configurations and input size. Any <a href="/docs/28.0.0/querying/granularities">granularity</a> can be provided here. Note that all segments in the same time chunk should have the same segment granularity.<br><br>Avoid <code>WEEK</code> granularity for data partitioning because weeks don't align neatly with months and years, making it difficult to change partitioning by coarser granularity. Instead, opt for other partitioning options such as <code>DAY</code> or <code>MONTH</code>, which offer more flexibility.</td><td><code>day</code></td></tr><tr><td>queryGranularity</td><td>The resolution of timestamp storage within each segment. This must be equal to, or finer, than <code>segmentGranularity</code>. This will be the finest granularity that you can query at and still receive sensible results, but note that you can still query at anything coarser than this granularity. E.g., a value of <code>minute</code> will mean that records will be stored at minutely granularity, and can be sensibly queried at any multiple of minutes (including minutely, 5-minutely, hourly, etc).<br><br>Any <a href="/docs/28.0.0/querying/granularities">granularity</a> can be provided here. Use <code>none</code> to store timestamps as-is, without any truncation. Note that <code>rollup</code> will be applied if it is set even when the <code>queryGranularity</code> is set to <code>none</code>.</td><td><code>none</code></td></tr><tr><td>rollup</td><td>Whether to use ingestion-time <a href="/docs/28.0.0/ingestion/rollup">rollup</a> or not. Note that rollup is still effective even when <code>queryGranularity</code> is set to <code>none</code>. Your data will be rolled up if they have the exactly same timestamp.</td><td><code>true</code></td></tr><tr><td>intervals</td><td>A list of intervals defining time chunks for segments. Specify interval values using ISO8601 format. For example, <code>["2021-12-06T21:27:10+00:00/2021-12-07T00:00:00+00:00"]</code>. If you omit the time, the time defaults to "00:00:00".<br><br>Druid breaks the list up and rounds off the list values based on the <code>segmentGranularity</code>.<br><br>If <code>null</code> or not provided, batch ingestion tasks generally determine which time chunks to output based on the timestamps found in the input data.<br><br>If specified, batch ingestion tasks may be able to skip a determining-partitions phase, which can result in faster ingestion. Batch ingestion tasks may also be able to request all their locks up-front instead of one by one. Batch ingestion tasks throw away any records with timestamps outside of the specified intervals.<br><br>Ignored for any form of streaming ingestion.</td><td><code>null</code></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="transformspec"><code>transformSpec</code><a href="#transformspec" class="hash-link" aria-label="Direct link to transformspec" title="Direct link to transformspec"></a></h3><p>The <code>transformSpec</code> is located in <code>dataSchema</code> → <code>transformSpec</code> and is responsible for transforming and filtering |
| records during ingestion time. It is optional. An example <code>transformSpec</code> is:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">"transformSpec": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "transforms": [</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> { "type": "expression", "name": "countryUpper", "expression": "upper(country)" }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> ],</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "filter": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "type": "selector",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "dimension": "country",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "value": "San Serriffe"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="theme-admonition theme-admonition-info alert alert--info admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg></span>info</div><div class="admonitionContent_S0QG"><p> Conceptually, after input data records are read, Druid applies ingestion spec components in a particular order: |
| first <a href="/docs/28.0.0/ingestion/data-formats#flattenspec"><code>flattenSpec</code></a> (if any), then <a href="#timestampspec"><code>timestampSpec</code></a>, then <a href="#transformspec"><code>transformSpec</code></a>, |
| and finally <a href="#dimensionsspec"><code>dimensionsSpec</code></a> and <a href="#metricsspec"><code>metricsSpec</code></a>. Keep this in mind when writing |
| your ingestion spec.</p></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="transforms">Transforms<a href="#transforms" class="hash-link" aria-label="Direct link to Transforms" title="Direct link to Transforms"></a></h4><p>The <code>transforms</code> list allows you to specify a set of expressions to evaluate on top of input data. Each transform has a |
| "name" which can be referred to by your <code>dimensionsSpec</code>, <code>metricsSpec</code>, etc.</p><p>If a transform has the same name as a field in an input row, then it will shadow the original field. Transforms that |
| shadow fields may still refer to the fields they shadow. This can be used to transform a field "in-place".</p><p>Transforms do have some limitations. They can only refer to fields present in the actual input rows; in particular, |
| they cannot refer to other transforms. And they cannot remove fields, only add them. However, they can shadow a field |
| with another field containing all nulls, which will act similarly to removing the field.</p><p>Druid currently includes one kind of built-in transform, the expression transform. It has the following syntax:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">{</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "type": "expression",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "name": "<output name>",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "expression": "<expr>"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The <code>expression</code> is a <a href="/docs/28.0.0/querying/math-expr">Druid query expression</a>.</p><div class="theme-admonition theme-admonition-info alert alert--info admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg></span>info</div><div class="admonitionContent_S0QG"><p> Conceptually, after input data records are read, Druid applies ingestion spec components in a particular order: |
| first <a href="/docs/28.0.0/ingestion/data-formats#flattenspec"><code>flattenSpec</code></a> (if any), then <a href="#timestampspec"><code>timestampSpec</code></a>, then <a href="#transformspec"><code>transformSpec</code></a>, |
| and finally <a href="#dimensionsspec"><code>dimensionsSpec</code></a> and <a href="#metricsspec"><code>metricsSpec</code></a>. Keep this in mind when writing |
| your ingestion spec.</p></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="filter">Filter<a href="#filter" class="hash-link" aria-label="Direct link to Filter" title="Direct link to Filter"></a></h4><p>The <code>filter</code> conditionally filters input rows during ingestion. Only rows that pass the filter will be |
| ingested. Any of Druid's standard <a href="/docs/28.0.0/querying/filters">query filters</a> can be used. Note that within a |
| <code>transformSpec</code>, the <code>transforms</code> are applied before the <code>filter</code>, so the filter can refer to a transform.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="legacy-dataschema-spec">Legacy <code>dataSchema</code> spec<a href="#legacy-dataschema-spec" class="hash-link" aria-label="Direct link to legacy-dataschema-spec" title="Direct link to legacy-dataschema-spec"></a></h3><div class="theme-admonition theme-admonition-info alert alert--info admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg></span>info</div><div class="admonitionContent_S0QG"><p> The <code>dataSchema</code> spec has been changed in 0.17.0. The new spec is supported by all ingestion methods |
| except for <em>Hadoop</em> ingestion. See <a href="#dataschema"><code>dataSchema</code></a> for the new spec.</p></div></div><p>The legacy <code>dataSchema</code> spec has below two more components in addition to the ones listed in the <a href="#dataschema"><code>dataSchema</code></a> section above.</p><ul><li><a href="#parser-deprecated">input row parser</a>, <a href="#flattenspec">flattening of nested data</a> (if needed)</li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="parser-deprecated"><code>parser</code> (Deprecated)<a href="#parser-deprecated" class="hash-link" aria-label="Direct link to parser-deprecated" title="Direct link to parser-deprecated"></a></h4><p>In legacy <code>dataSchema</code>, the <code>parser</code> is located in the <code>dataSchema</code> → <code>parser</code> and is responsible for configuring a wide variety of |
| items related to parsing input records. The <code>parser</code> is deprecated and it is highly recommended to use <code>inputFormat</code> instead. |
| For details about <code>inputFormat</code> and supported <code>parser</code> types, see the <a href="/docs/28.0.0/ingestion/data-formats">"Data formats" page</a>.</p><p>For details about major components of the <code>parseSpec</code>, refer to their subsections:</p><ul><li><a href="#timestampspec"><code>timestampSpec</code></a>, responsible for configuring the <a href="/docs/28.0.0/ingestion/schema-model#primary-timestamp">primary timestamp</a>.</li><li><a href="#dimensionsspec"><code>dimensionsSpec</code></a>, responsible for configuring <a href="/docs/28.0.0/ingestion/schema-model#dimensions">dimensions</a>.</li><li><a href="#flattenspec"><code>flattenSpec</code></a>, responsible for flattening nested data formats.</li></ul><p>An example <code>parser</code> is:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">"parser": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "type": "string",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "parseSpec": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "format": "json",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "flattenSpec": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "useFieldDiscovery": true,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "fields": [</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> { "type": "path", "name": "userId", "expr": "$.user.id" }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> ]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "timestampSpec": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "column": "timestamp",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "format": "auto"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "dimensionsSpec": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "dimensions": [</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "page",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "language",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> { "type": "long", "name": "userId" }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> ]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="flattenspec"><code>flattenSpec</code><a href="#flattenspec" class="hash-link" aria-label="Direct link to flattenspec" title="Direct link to flattenspec"></a></h4><p>In the legacy <code>dataSchema</code>, the <code>flattenSpec</code> is located in <code>dataSchema</code> → <code>parser</code> → <code>parseSpec</code> → <code>flattenSpec</code> and is responsible for |
| bridging the gap between potentially nested input data (such as JSON, Avro, etc) and Druid's flat data model. |
| See <a href="/docs/28.0.0/ingestion/data-formats#flattenspec">Flatten spec</a> for more details.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="ioconfig"><code>ioConfig</code><a href="#ioconfig" class="hash-link" aria-label="Direct link to ioconfig" title="Direct link to ioconfig"></a></h2><p>The <code>ioConfig</code> influences how data is read from a source system, such as Apache Kafka, Amazon S3, a mounted |
| filesystem, or any other supported source system. The <code>inputFormat</code> property applies to all |
| <a href="/docs/28.0.0/ingestion/#ingestion-methods">ingestion method</a> except for Hadoop ingestion. The Hadoop ingestion still |
| uses the <a href="#parser-deprecated"><code>parser</code></a> in the legacy <code>dataSchema</code>. |
| The rest of <code>ioConfig</code> is specific to each individual ingestion method. |
| An example <code>ioConfig</code> to read JSON data is:</p><div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token property">"ioConfig"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"<ingestion-method-specific type code>"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"inputFormat"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"json"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>For more details, see the documentation provided by each <a href="/docs/28.0.0/ingestion/#ingestion-methods">ingestion method</a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="tuningconfig"><code>tuningConfig</code><a href="#tuningconfig" class="hash-link" aria-label="Direct link to tuningconfig" title="Direct link to tuningconfig"></a></h2><p>Tuning properties are specified in a <code>tuningConfig</code>, which goes at the top level of an ingestion spec. Some |
| properties apply to all <a href="/docs/28.0.0/ingestion/#ingestion-methods">ingestion methods</a>, but most are specific to each individual |
| ingestion method. An example <code>tuningConfig</code> that sets all of the shared, common properties to their defaults |
| is:</p><div class="language-plaintext codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-plaintext codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token plain">"tuningConfig": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "type": "<ingestion-method-specific type code>",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "maxRowsInMemory": 1000000,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "maxBytesInMemory": <one-sixth of JVM memory>,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "indexSpec": {</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "bitmap": { "type": "roaring" },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "dimensionCompression": "lz4",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "metricCompression": "lz4",</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> "longEncoding": "longs"</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> },</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> <other ingestion-method-specific properties></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><table><thead><tr><th>Field</th><th>Description</th><th>Default</th></tr></thead><tbody><tr><td>type</td><td>Each ingestion method has its own tuning type code. You must specify the type code that matches your ingestion method. Common options are <code>index</code>, <code>hadoop</code>, <code>kafka</code>, and <code>kinesis</code>.</td><td></td></tr><tr><td>maxRowsInMemory</td><td>The maximum number of records to store in memory before persisting to disk. Note that this is the number of rows post-rollup, and so it may not be equal to the number of input records. Ingested records will be persisted to disk when either <code>maxRowsInMemory</code> or <code>maxBytesInMemory</code> are reached (whichever happens first).</td><td><code>1000000</code></td></tr><tr><td>maxBytesInMemory</td><td>The maximum aggregate size of records, in bytes, to store in the JVM heap before persisting. This is based on a rough estimate of memory usage. Ingested records will be persisted to disk when either <code>maxRowsInMemory</code> or <code>maxBytesInMemory</code> are reached (whichever happens first). <code>maxBytesInMemory</code> also includes heap usage of artifacts created from intermediary persists. This means that after every persist, the amount of <code>maxBytesInMemory</code> until the next persist will decrease. If the sum of bytes of all intermediary persisted artifacts exceeds <code>maxBytesInMemory</code> the task fails.<br><br>Setting <code>maxBytesInMemory</code> to -1 disables this check, meaning Druid will rely entirely on <code>maxRowsInMemory</code> to control memory usage. Setting it to zero means the default value will be used (one-sixth of JVM heap size).<br><br>Note that the estimate of memory usage is designed to be an overestimate, and can be especially high when using complex ingest-time aggregators, including sketches. If this causes your indexing workloads to persist to disk too often, you can set <code>maxBytesInMemory</code> to -1 and rely on <code>maxRowsInMemory</code> instead.</td><td>One-sixth of max JVM heap size</td></tr><tr><td>skipBytesInMemoryOverheadCheck</td><td>The calculation of maxBytesInMemory takes into account overhead objects created during ingestion and each intermediate persist. Setting this to true can exclude the bytes of these overhead objects from maxBytesInMemory check.</td><td>false</td></tr><tr><td>indexSpec</td><td>Defines segment storage format options to use at indexing time.</td><td>See <a href="#indexspec"><code>indexSpec</code></a> for more information.</td></tr><tr><td>indexSpecForIntermediatePersists</td><td>Defines segment storage format options to use at indexing time for intermediate persisted temporary segments.</td><td>See <a href="#indexspec"><code>indexSpec</code></a> for more information.</td></tr><tr><td>Other properties</td><td>Each ingestion method has its own list of additional tuning properties. See the documentation for each method for a full list: <a href="/docs/28.0.0/development/extensions-core/kafka-supervisor-reference#supervisor-tuning-configuration">Kafka indexing service</a>, <a href="/docs/28.0.0/development/extensions-core/kinesis-ingestion#supervisor-tuning-configuration">Kinesis indexing service</a>, <a href="/docs/28.0.0/ingestion/native-batch#tuningconfig">Native batch</a>, and <a href="/docs/28.0.0/ingestion/hadoop#tuningconfig">Hadoop-based</a>.</td><td></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="indexspec"><code>indexSpec</code><a href="#indexspec" class="hash-link" aria-label="Direct link to indexspec" title="Direct link to indexspec"></a></h3><p>The <code>indexSpec</code> object can include the following properties:</p><table><thead><tr><th>Field</th><th>Description</th><th>Default</th></tr></thead><tbody><tr><td>bitmap</td><td>Compression format for bitmap indexes. Should be a JSON object with <code>type</code> set to <code>roaring</code> or <code>concise</code>.</td><td><code>{"type": "roaring"}</code></td></tr><tr><td>dimensionCompression</td><td>Compression format for dimension columns. Options are <code>lz4</code>, <code>lzf</code>, <code>zstd</code>, or <code>uncompressed</code>.</td><td><code>lz4</code></td></tr><tr><td>stringDictionaryEncoding</td><td>Encoding format for STRING value dictionaries used by STRING and COMPLEX<!-- --><<!-- -->json<!-- -->><!-- --> columns. <br><br>Example to enable front coding: <code>{"type":"frontCoded", "bucketSize": 4}</code><br><code>bucketSize</code> is the number of values to place in a bucket to perform delta encoding. Must be a power of 2, maximum is 128. Defaults to 4.<br> <code>formatVersion</code> can specify older versions for backwards compatibility during rolling upgrades, valid options are <code>0</code> and <code>1</code>. Defaults to <code>0</code> for backwards compatibility.<br><br>See <a href="#front-coding">Front coding</a> for more information.</td><td><code>{"type":"utf8"}</code></td></tr><tr><td>metricCompression</td><td>Compression format for primitive type metric columns. Options are <code>lz4</code>, <code>lzf</code>, <code>zstd</code>, <code>uncompressed</code>, or <code>none</code> (which is more efficient than <code>uncompressed</code>, but not supported by older versions of Druid).</td><td><code>lz4</code></td></tr><tr><td>longEncoding</td><td>Encoding format for long-typed columns. Applies regardless of whether they are dimensions or metrics. Options are <code>auto</code> or <code>longs</code>. <code>auto</code> encodes the values using offset or lookup table depending on column cardinality, and store them with variable size. <code>longs</code> stores the value as-is with 8 bytes each.</td><td><code>longs</code></td></tr><tr><td>jsonCompression</td><td>Compression format to use for nested column raw data. Options are <code>lz4</code>, <code>lzf</code>, <code>zstd</code>, or <code>uncompressed</code>.</td><td><code>lz4</code></td></tr></tbody></table><h5 class="anchor anchorWithStickyNavbar_LWe7" id="front-coding">Front coding<a href="#front-coding" class="hash-link" aria-label="Direct link to Front coding" title="Direct link to Front coding"></a></h5><p>Front coding is an experimental feature starting in version 25.0. Front coding is an incremental encoding strategy that Druid can use to store STRING and <a href="/docs/28.0.0/querying/nested-columns">COMPLEX<!-- --><<!-- -->json<!-- -->></a> columns. It allows Druid to create smaller UTF-8 encoded segments with very little performance cost.</p><p>You can enable front coding with all types of ingestion. For information on defining an <code>indexSpec</code> in a query context, see <a href="/docs/28.0.0/multi-stage-query/reference#context-parameters">SQL-based ingestion reference</a>.</p><div class="theme-admonition theme-admonition-info alert alert--info admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg></span>info</div><div class="admonitionContent_S0QG"><p> Front coding was originally introduced in Druid 25.0, and an improved 'version 1' was introduced in Druid 26.0, with typically faster read speed and smaller storage size. The current recommendation is to enable it in a staging environment and fully test your use case before using in production. By default, segments created with front coding enabled in Druid 26.0 are backwards compatible with Druid 25.0, but those created with Druid 26.0 or 25.0 are not compatible with Druid versions older than 25.0. If using front coding in Druid 25.0 and upgrading to Druid 26.0, the <code>formatVersion</code> defaults to <code>0</code> to keep writing out the older format to enable seamless downgrades to Druid 25.0, and then later is recommended to be changed to <code>1</code> once determined that rollback is not necessary.</p></div></div><p>Beyond these properties, each ingestion method has its own specific tuning properties. See the documentation for each |
| <a href="/docs/28.0.0/ingestion/#ingestion-methods">ingestion method</a> for details.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/docs/28.0.0/ingestion/hadoop"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Hadoop-based</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/docs/28.0.0/ingestion/schema-design"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Schema design tips</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#dataschema" class="table-of-contents__link toc-highlight"><code>dataSchema</code></a><ul><li><a href="#datasource" class="table-of-contents__link toc-highlight"><code>dataSource</code></a></li><li><a href="#timestampspec" class="table-of-contents__link toc-highlight"><code>timestampSpec</code></a></li><li><a href="#dimensionsspec" class="table-of-contents__link toc-highlight"><code>dimensionsSpec</code></a></li><li><a href="#metricsspec" class="table-of-contents__link toc-highlight"><code>metricsSpec</code></a></li><li><a href="#granularityspec" class="table-of-contents__link toc-highlight"><code>granularitySpec</code></a></li><li><a href="#transformspec" class="table-of-contents__link toc-highlight"><code>transformSpec</code></a></li><li><a href="#legacy-dataschema-spec" class="table-of-contents__link toc-highlight">Legacy <code>dataSchema</code> spec</a></li></ul></li><li><a href="#ioconfig" class="table-of-contents__link toc-highlight"><code>ioConfig</code></a></li><li><a href="#tuningconfig" class="table-of-contents__link toc-highlight"><code>tuningConfig</code></a><ul><li><a href="#indexspec" class="table-of-contents__link toc-highlight"><code>indexSpec</code></a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer"><div class="container container-fluid"><div class="footer__bottom text--center"><div class="margin-bottom--sm"><img src="/img/favicon.png" class="themedImage_ToTc themedImage--light_HNdA footer__logo"><img src="/img/favicon.png" class="themedImage_ToTc themedImage--dark_i4oU footer__logo"></div><div class="footer__copyright">Copyright © 2023 Apache Software Foundation. Except where otherwise noted, licensed under CC BY-SA 4.0. Apache Druid, Druid, and the Druid logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</div></div></div></footer></div> |
| <script src="/assets/js/runtime~main.0dcbfdea.js"></script> |
| <script src="/assets/js/main.7f6fdf81.js"></script> |
| </body> |
| </html> |
