| <!doctype html> |
| <html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-current plugin-docs plugin-id-default docs-doc-id-ingestion/schema-design"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="generator" content="Docusaurus v2.4.1"> |
| <title data-rh="true">Schema design tips | Apache® Druid</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:image" content="https://druid.apache.org/img/druid_nav.png"><meta data-rh="true" name="twitter:image" content="https://druid.apache.org/img/druid_nav.png"><meta data-rh="true" property="og:url" content="https://druid.apache.org/docs/26.0.0/ingestion/schema-design"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="current"><meta data-rh="true" name="docusaurus_tag" content="docs-default-current"><meta data-rh="true" name="docsearch:version" content="current"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-current"><meta data-rh="true" property="og:title" content="Schema design tips | Apache® Druid"><meta data-rh="true" name="description" content="<!--"><meta data-rh="true" property="og:description" content="<!--"><link data-rh="true" rel="icon" href="/img/favicon.png"><link data-rh="true" rel="canonical" href="https://druid.apache.org/docs/26.0.0/ingestion/schema-design"><link data-rh="true" rel="alternate" href="https://druid.apache.org/docs/26.0.0/ingestion/schema-design" hreflang="en"><link data-rh="true" rel="alternate" href="https://druid.apache.org/docs/26.0.0/ingestion/schema-design" hreflang="x-default"><link rel="preconnect" href="https://www.google-analytics.com"> |
| <link rel="preconnect" href="https://www.googletagmanager.com"> |
| <script async src="https://www.googletagmanager.com/gtag/js?id=UA-131010415-1"></script> |
| <script>function gtag(){dataLayer.push(arguments)}window.dataLayer=window.dataLayer||[],gtag("js",new Date),gtag("config","UA-131010415-1",{})</script> |
| |
| |
| |
| |
| <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"> |
| <script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><link rel="stylesheet" href="/assets/css/styles.f80751b3.css"> |
| <link rel="preload" href="/assets/js/runtime~main.38900cbf.js" as="script"> |
| <link rel="preload" href="/assets/js/main.5e106d68.js" as="script"> |
| </head> |
| <body class="navigation-with-keyboard"> |
| <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus"> |
| <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><nav aria-label="Main" class="navbar navbar--fixed-top navbar--dark"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/druid_nav.png" alt="Apache® Druid" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/druid_nav.png" alt="Apache® Druid" class="themedImage_ToTc themedImage--dark_i4oU"></div></a></div><div class="navbar__items navbar__items--right"><a class="navbar__item navbar__link" href="/technology">Technology</a><a class="navbar__item navbar__link" href="/use-cases">Use Cases</a><a class="navbar__item navbar__link" href="/druid-powered">Powered By</a><a class="navbar__item navbar__link" href="/docs/26.0.0/design/">Docs</a><a class="navbar__item navbar__link" href="/community/">Community</a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Apache®</a><ul class="dropdown__menu"><li><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Foundation<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://apachecon.com/?ref=druid.apache.org" target="_blank" rel="noopener noreferrer" class="dropdown__link">Events<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/licenses/" target="_blank" rel="noopener noreferrer" class="dropdown__link">License<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Thanks<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/security/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Security<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Sponsorship<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a class="navbar__item navbar__link" href="/downloads/">Download</a><div class="searchBox_ZlJk"><div class="navbar__search"><span aria-label="expand searchbar" role="button" class="search-icon" tabindex="0"></span><input type="search" id="search_input_react" placeholder="Loading..." aria-label="Search" class="navbar__search-input search-bar" disabled=""></div></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/26.0.0/design/">Getting started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/26.0.0/tutorials/tutorial-batch">Tutorials</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/26.0.0/design/architecture">Design</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/docs/26.0.0/ingestion/">Ingestion</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/ingestion/">Ingestion</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/ingestion/data-formats">Data formats</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/ingestion/data-model">Data model</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/ingestion/rollup">Data rollup</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/ingestion/partitioning">Partitioning</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/ingestion/ingestion-spec">Ingestion spec</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/docs/26.0.0/ingestion/schema-design">Schema design tips</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/26.0.0/development/extensions-core/kafka-ingestion">Stream ingestion</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/26.0.0/ingestion/native-batch">Batch ingestion</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/26.0.0/multi-stage-query/">SQL-based ingestion 🆕</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/ingestion/tasks">Task reference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/26.0.0/ingestion/faq">Troubleshooting FAQ</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/26.0.0/data-management/">Data management</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/26.0.0/querying/sql">Querying</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/26.0.0/configuration/">Configuration</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/26.0.0/operations/web-console">Operations</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/26.0.0/development/overview">Development</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/26.0.0/misc/papers-and-talks">Misc</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Ingestion</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Schema design tips</span><meta itemprop="position" content="2"></li></ul></nav><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Schema design tips</h1></header><h2 class="anchor anchorWithStickyNavbar_LWe7" id="druids-data-model">Druid's data model<a href="#druids-data-model" class="hash-link" aria-label="Direct link to Druid's data model" title="Direct link to Druid's data model"></a></h2><p>For general information, check out the documentation on <a href="/docs/26.0.0/ingestion/data-model">Druid's data model</a> on the main |
| ingestion overview page. The rest of this page discusses tips for users coming from other kinds of systems, as well as |
| general tips and common practices.</p><ul><li>Druid data is stored in <a href="/docs/26.0.0/ingestion/data-model">datasources</a>, which are similar to tables in a traditional RDBMS.</li><li>Druid datasources can be ingested with or without <a href="/docs/26.0.0/ingestion/rollup">rollup</a>. With rollup enabled, Druid partially aggregates your data during ingestion, potentially reducing its row count, decreasing storage footprint, and improving query performance. With rollup disabled, Druid stores one row for each row in your input data, without any pre-aggregation.</li><li>Every row in Druid must have a timestamp. Data is always partitioned by time, and every query has a time filter. Query results can also be broken down by time buckets like minutes, hours, days, and so on.</li><li>All columns in Druid datasources, other than the timestamp column, are either dimensions or metrics. This follows the <a href="https://en.wikipedia.org/wiki/Online_analytical_processing#Overview_of_OLAP_systems" target="_blank" rel="noopener noreferrer">standard naming convention</a> of OLAP data.</li><li>Typical production datasources have tens to hundreds of columns.</li><li><a href="/docs/26.0.0/ingestion/data-model#dimensions">Dimension columns</a> are stored as-is, so they can be filtered on, grouped by, or aggregated at query time. They are always single Strings, <a href="/docs/26.0.0/querying/multi-value-dimensions">arrays of Strings</a>, single Longs, single Doubles or single Floats.</li><li><a href="/docs/26.0.0/ingestion/data-model#metrics">Metric columns</a> are stored <a href="/docs/26.0.0/querying/aggregations">pre-aggregated</a>, so they can only be aggregated at query time (not filtered or grouped by). They are often stored as numbers (integers or floats) but can also be stored as complex objects like <a href="/docs/26.0.0/querying/aggregations#approximate-aggregations">HyperLogLog sketches or approximate quantile sketches</a>. Metrics can be configured at ingestion time even when rollup is disabled, but are most useful when rollup is enabled.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="if-youre-coming-from-a">If you're coming from a<a href="#if-youre-coming-from-a" class="hash-link" aria-label="Direct link to If you're coming from a" title="Direct link to If you're coming from a"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="relational-model">Relational model<a href="#relational-model" class="hash-link" aria-label="Direct link to Relational model" title="Direct link to Relational model"></a></h3><p>(Like Hive or PostgreSQL.)</p><p>Druid datasources are generally equivalent to tables in a relational database. Druid <a href="/docs/26.0.0/querying/lookups">lookups</a> |
| can act similarly to data-warehouse-style dimension tables, but as you'll see below, denormalization is often |
| recommended if you can get away with it.</p><p>Common practice for relational data modeling involves <a href="https://en.wikipedia.org/wiki/Database_normalization" target="_blank" rel="noopener noreferrer">normalization</a>: |
| the idea of splitting up data into multiple tables such that data redundancy is reduced or eliminated. For example, in a |
| "sales" table, best-practices relational modeling calls for a "product id" column that is a foreign key into a separate |
| "products" table, which in turn has "product id", "product name", and "product category" columns. This prevents the |
| product name and category from needing to be repeated on different rows in the "sales" table that refer to the same |
| product.</p><p>In Druid, on the other hand, it is common to use totally flat datasources that do not require joins at query time. In |
| the example of the "sales" table, in Druid it would be typical to store "product<em>id", "product_name", and |
| "product_category" as dimensions directly in a Druid "sales" datasource, without using a separate "products" table. |
| Totally flat schemas substantially increase performance, since the need for joins is eliminated at query time. As an |
| an added speed boost, this also allows Druid's query layer to operate directly on compressed dictionary-encoded data. |
| Perhaps counter-intuitively, this does _not</em> substantially increase storage footprint relative to normalized schemas, |
| since Druid uses dictionary encoding to effectively store just a single integer per row for string columns.</p><p>If necessary, Druid datasources can be partially normalized through the use of <a href="/docs/26.0.0/querying/lookups">lookups</a>, |
| which are the rough equivalent of dimension tables in a relational database. At query time, you would use Druid's SQL |
| <code>LOOKUP</code> function, or native lookup extraction functions, instead of using the JOIN keyword like you would in a |
| relational database. Since lookup tables impose an increase in memory footprint and incur more computational overhead |
| at query time, it is only recommended to do this if you need the ability to update a lookup table and have the changes |
| reflected immediately for already-ingested rows in your main table.</p><p>Tips for modeling relational data in Druid:</p><ul><li>Druid datasources do not have primary or unique keys, so skip those.</li><li>Denormalize if possible. If you need to be able to update dimension / lookup tables periodically and have those |
| changes reflected in already-ingested data, consider partial normalization with <a href="/docs/26.0.0/querying/lookups">lookups</a>.</li><li>If you need to join two large distributed tables with each other, you must do this before loading the data into Druid. |
| Druid does not support query-time joins of two datasources. Lookups do not help here, since a full copy of each lookup |
| table is stored on each Druid server, so they are not a good choice for large tables.</li><li>Consider whether you want to enable <a href="#rollup">rollup</a> for pre-aggregation, or whether you want to disable |
| rollup and load your existing data as-is. Rollup in Druid is similar to creating a summary table in a relational model.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="time-series-model">Time series model<a href="#time-series-model" class="hash-link" aria-label="Direct link to Time series model" title="Direct link to Time series model"></a></h3><p>(Like OpenTSDB or InfluxDB.)</p><p>Similar to time series databases, Druid's data model requires a timestamp. Druid is not a timeseries database, but |
| it is a natural choice for storing timeseries data. Its flexible data model allows it to store both timeseries and |
| non-timeseries data, even in the same datasource.</p><p>To achieve best-case compression and query performance in Druid for timeseries data, it is important to partition and |
| sort by metric name, like timeseries databases often do. See <a href="/docs/26.0.0/ingestion/partitioning">Partitioning and sorting</a> for more details.</p><p>Tips for modeling timeseries data in Druid:</p><ul><li>Druid does not think of data points as being part of a "time series". Instead, Druid treats each point separately |
| for ingestion and aggregation.</li><li>Create a dimension that indicates the name of the series that a data point belongs to. This dimension is often called |
| "metric" or "name". Do not get the dimension named "metric" confused with the concept of Druid metrics. Place this |
| first in the list of dimensions in your "dimensionsSpec" for best performance (this helps because it improves locality; |
| see <a href="/docs/26.0.0/ingestion/partitioning">partitioning and sorting</a> below for details).</li><li>Create other dimensions for attributes attached to your data points. These are often called "tags" in timeseries |
| database systems.</li><li>Create <a href="/docs/26.0.0/querying/aggregations">metrics</a> corresponding to the types of aggregations that you want to be able |
| to query. Typically this includes "sum", "min", and "max" (in one of the long, float, or double flavors). If you want the ability |
| to compute percentiles or quantiles, use Druid's <a href="/docs/26.0.0/querying/aggregations#approximate-aggregations">approximate aggregators</a>.</li><li>Consider enabling <a href="/docs/26.0.0/ingestion/rollup">rollup</a>, which will allow Druid to potentially combine multiple points into one |
| row in your Druid datasource. This can be useful if you want to store data at a different time granularity than it is |
| naturally emitted. It is also useful if you want to combine timeseries and non-timeseries data in the same datasource.</li><li>If you don't know ahead of time what columns you'll want to ingest, use an empty dimensions list to trigger |
| <a href="#schema-auto-discovery-for-dimensions">automatic detection of dimension columns</a>.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="log-aggregation-model">Log aggregation model<a href="#log-aggregation-model" class="hash-link" aria-label="Direct link to Log aggregation model" title="Direct link to Log aggregation model"></a></h3><p>(Like Elasticsearch or Splunk.)</p><p>Similar to log aggregation systems, Druid offers inverted indexes for fast searching and filtering. Druid's search |
| capabilities are generally less developed than these systems, and its analytical capabilities are generally more |
| developed. The main data modeling differences between Druid and these systems are that when ingesting data into Druid, |
| you must be more explicit. Druid columns have types specific upfront.</p><p>Tips for modeling log data in Druid:</p><ul><li>If you don't know ahead of time what columns to ingest, you can have Druid perform <a href="#schema-auto-discovery-for-dimensions">schema auto-discovery</a>.</li><li>If you have nested data, you can ingest it using the <a href="/docs/26.0.0/querying/nested-columns">nested columns</a> feature or flatten it using a <a href="/docs/26.0.0/ingestion/ingestion-spec#flattenspec"><code>flattenSpec</code></a>.</li><li>Consider enabling <a href="/docs/26.0.0/ingestion/rollup">rollup</a> if you have mainly analytical use cases for your log data. This will |
| mean you lose the ability to retrieve individual events from Druid, but you potentially gain substantial compression and |
| query performance boosts.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="general-tips-and-best-practices">General tips and best practices<a href="#general-tips-and-best-practices" class="hash-link" aria-label="Direct link to General tips and best practices" title="Direct link to General tips and best practices"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="rollup">Rollup<a href="#rollup" class="hash-link" aria-label="Direct link to Rollup" title="Direct link to Rollup"></a></h3><p>Druid can roll up data as it is ingested to minimize the amount of raw data that needs to be stored. This is a form |
| of summarization or pre-aggregation. For more details, see the <a href="/docs/26.0.0/ingestion/rollup">Rollup</a> section of the ingestion |
| documentation.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="partitioning-and-sorting">Partitioning and sorting<a href="#partitioning-and-sorting" class="hash-link" aria-label="Direct link to Partitioning and sorting" title="Direct link to Partitioning and sorting"></a></h3><p>Optimally partitioning and sorting your data can have substantial impact on footprint and performance. For more details, |
| see the <a href="/docs/26.0.0/ingestion/partitioning">Partitioning</a> section of the ingestion documentation.</p><a name="sketches"></a><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sketches-for-high-cardinality-columns">Sketches for high cardinality columns<a href="#sketches-for-high-cardinality-columns" class="hash-link" aria-label="Direct link to Sketches for high cardinality columns" title="Direct link to Sketches for high cardinality columns"></a></h3><p>When dealing with high cardinality columns like user IDs or other unique IDs, consider using sketches for approximate |
| analysis rather than operating on the actual values. When you ingest data using a sketch, Druid does not store the |
| original raw data, but instead stores a "sketch" of it that it can feed into a later computation at query time. Popular |
| use cases for sketches include count-distinct and quantile computation. Each sketch is designed for just one particular |
| kind of computation.</p><p>In general using sketches serves two main purposes: improving rollup, and reducing memory footprint at |
| query time.</p><p>Sketches improve rollup ratios because they allow you to collapse multiple distinct values into the same sketch. For |
| example, if you have two rows that are identical except for a user ID (perhaps two users did the same action at the |
| same time), storing them in a count-distinct sketch instead of as-is means you can store the data in one row instead of |
| two. You won't be able to retrieve the user IDs or compute exact distinct counts, but you'll still be able to compute |
| approximate distinct counts, and you'll reduce your storage footprint.</p><p>Sketches reduce memory footprint at query time because they limit the amount of data that needs to be shuffled between |
| servers. For example, in a quantile computation, instead of needing to send all data points to a central location |
| so they can be sorted and the quantile can be computed, Druid instead only needs to send a sketch of the points. This |
| can reduce data transfer needs to mere kilobytes.</p><p>For details about the sketches available in Druid, see the |
| <a href="/docs/26.0.0/querying/aggregations#approximate-aggregations">approximate aggregators</a> page.</p><p>If you prefer videos, take a look at <a href="https://www.youtube.com/watch?v=Hpd3f_MLdXo" target="_blank" rel="noopener noreferrer">Not exactly!</a>, a conference talk |
| about sketches in Druid.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="string-vs-numeric-dimensions">String vs numeric dimensions<a href="#string-vs-numeric-dimensions" class="hash-link" aria-label="Direct link to String vs numeric dimensions" title="Direct link to String vs numeric dimensions"></a></h3><p>If the user wishes to ingest a column as a numeric-typed dimension (Long, Double or Float), it is necessary to specify the type of the column in the <code>dimensions</code> section of the <code>dimensionsSpec</code>. If the type is omitted, Druid will ingest a column as the default String type.</p><p>There are performance tradeoffs between string and numeric columns. Numeric columns are generally faster to group on |
| than string columns. But unlike string columns, numeric columns don't have indexes, so they can be slower to filter on. |
| You may want to experiment to find the optimal choice for your use case.</p><p>For details about how to configure numeric dimensions, see the <a href="/docs/26.0.0/ingestion/ingestion-spec#dimensionsspec"><code>dimensionsSpec</code></a> documentation.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="secondary-timestamps">Secondary timestamps<a href="#secondary-timestamps" class="hash-link" aria-label="Direct link to Secondary timestamps" title="Direct link to Secondary timestamps"></a></h3><p>Druid schemas must always include a primary timestamp. The primary timestamp is used for |
| <a href="/docs/26.0.0/ingestion/partitioning">partitioning and sorting</a> your data, so it should be the timestamp that you will most often filter on. |
| Druid is able to rapidly identify and retrieve data corresponding to time ranges of the primary timestamp column.</p><p>If your data has more than one timestamp, you can ingest the others as secondary timestamps. The best way to do this |
| is to ingest them as <a href="/docs/26.0.0/ingestion/ingestion-spec#dimensionsspec">long-typed dimensions</a> in milliseconds format. |
| If necessary, you can get them into this format using a <a href="/docs/26.0.0/ingestion/ingestion-spec#transformspec"><code>transformSpec</code></a> and |
| <a href="/docs/26.0.0/misc/math-expr">expressions</a> like <code>timestamp_parse</code>, which returns millisecond timestamps.</p><p>At query time, you can query secondary timestamps with <a href="/docs/26.0.0/querying/sql-scalar#date-and-time-functions">SQL time functions</a> |
| like <code>MILLIS_TO_TIMESTAMP</code>, <code>TIME_FLOOR</code>, and others. If you're using native Druid queries, you can use |
| <a href="/docs/26.0.0/misc/math-expr">expressions</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="nested-dimensions">Nested dimensions<a href="#nested-dimensions" class="hash-link" aria-label="Direct link to Nested dimensions" title="Direct link to Nested dimensions"></a></h3><p>You can ingest and store nested data in a Druid column as a <code>COMPLEX<json></code> data type. See <a href="/docs/26.0.0/querying/nested-columns">Nested columns</a> for more information.</p><p>If you want to ingest nested data in a format unsupported by the nested columns feature, you must use the <code>flattenSpec</code> object to flatten it. For example, if you have data of the following form:</p><div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"> </span><span class="token property">"foo"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"> </span><span class="token property">"bar"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">3</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>then before indexing it, you should transform it to:</p><div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"> </span><span class="token property">"foo_bar"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">3</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>See the <a href="/docs/26.0.0/ingestion/ingestion-spec#flattenspec"><code>flattenSpec</code></a> documentation for more details.</p><a name="counting"></a><h3 class="anchor anchorWithStickyNavbar_LWe7" id="counting-the-number-of-ingested-events">Counting the number of ingested events<a href="#counting-the-number-of-ingested-events" class="hash-link" aria-label="Direct link to Counting the number of ingested events" title="Direct link to Counting the number of ingested events"></a></h3><p>When rollup is enabled, count aggregators at query time do not actually tell you the number of rows that have been |
| ingested. They tell you the number of rows in the Druid datasource, which may be smaller than the number of rows |
| ingested.</p><p>In this case, a count aggregator at <em>ingestion</em> time can be used to count the number of events. However, it is important to note |
| that when you query for this metric, you should use a <code>longSum</code> aggregator. A <code>count</code> aggregator at query time will return |
| the number of Druid rows for the time interval, which can be used to determine what the roll-up ratio was.</p><p>To clarify with an example, if your ingestion spec contains:</p><div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token property">"metricsSpec"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"count"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token property">"name"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"count"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>You should query for the number of ingested rows with:</p><div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token property">"aggregations"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"longSum"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token property">"name"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"numIngestedEvents"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token property">"fieldName"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"count"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="schema-auto-discovery-for-dimensions">Schema auto-discovery for dimensions<a href="#schema-auto-discovery-for-dimensions" class="hash-link" aria-label="Direct link to Schema auto-discovery for dimensions" title="Direct link to Schema auto-discovery for dimensions"></a></h3><p>Druid can infer the schema for your data in one of two ways:</p><ul><li><a href="#type-aware-schema-discovery">Type-aware schema discovery (experimental)</a> where Druid infers the schema and type for your data. Type-aware schema discovery is an experimental feature currently available for native batch and streaming ingestion.</li><li><a href="#string-based-schema-discovery">String-based schema discovery</a> where all the discovered columns are typed as either native string or multi-value string columns.</li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="type-aware-schema-discovery">Type-aware schema discovery<a href="#type-aware-schema-discovery" class="hash-link" aria-label="Direct link to Type-aware schema discovery" title="Direct link to Type-aware schema discovery"></a></h4><blockquote><p>Note that using type-aware schema discovery can impact downstream BI tools depending on how they handle ARRAY typed columns.</p></blockquote><p>You can have Druid infer the schema and types for your data partially or fully by setting <code>dimensionsSpec.useSchemaDiscovery</code> to <code>true</code> and defining some or no dimensions in the dimensions list. </p><p>When performing type-aware schema discovery, Druid can discover all of the columns of your input data (that aren't in |
| the exclusion list). Druid automatically chooses the most appropriate native Druid type among <code>STRING</code>, <code>LONG</code>, |
| <code>DOUBLE</code>, <code>ARRAY<STRING></code>, <code>ARRAY<LONG></code>, <code>ARRAY<DOUBLE></code>, or <code>COMPLEX<json></code> for nested data. For input formats with |
| native boolean types, Druid ingests these values as strings if <code>druid.expressions.useStrictBooleans</code> is set to <code>false</code> |
| (the default), or longs if set to <code>true</code> (for more SQL compatible behavior). Array typed columns can be queried using |
| the <a href="/docs/26.0.0/querying/sql-array-functions">array functions</a> or <a href="/docs/26.0.0/querying/sql-functions#unnest">UNNEST</a>. Nested |
| columns can be queried with the <a href="/docs/26.0.0/querying/sql-json-functions">JSON functions</a>.</p><p>Mixed type columns are stored in the <em>least</em> restrictive type that can represent all values in the column. For example:</p><ul><li>Mixed numeric columns are <code>DOUBLE</code></li><li>If there are any strings present, then the column is a <code>STRING</code></li><li>If there are arrays, then the column becomes an array with the least restrictive element type</li><li>Any nested data or arrays of nested data become <code>COMPLEX<json></code> nested columns.</li></ul><p>If you're already using string-based schema discovery and want to migrate, see <a href="#migrating-to-type-aware-schema-discovery">Migrating to type-aware schema discovery</a>.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="string-based-schema-discovery">String-based schema discovery<a href="#string-based-schema-discovery" class="hash-link" aria-label="Direct link to String-based schema discovery" title="Direct link to String-based schema discovery"></a></h4><p>If you do not set <code>dimensionsSpec.useSchemaDiscovery</code> to <code>true</code>, Druid can still use the string-based schema discovery for ingestion if any of the following conditions are met: </p><ul><li>The dimension list is empty </li><li>You set <code>includeAllDimensions</code> to <code>true</code> </li></ul><p>Druid coerces primitives and arrays of primitive types into the native Druid string type. Nested data structures and arrays of nested data structures are ignored and not ingested.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="migrating-to-type-aware-schema-discovery">Migrating to type-aware schema discovery<a href="#migrating-to-type-aware-schema-discovery" class="hash-link" aria-label="Direct link to Migrating to type-aware schema discovery" title="Direct link to Migrating to type-aware schema discovery"></a></h4><p>If you previously used string-based schema discovery and want to migrate to type-aware schema discovery, do the following:</p><ul><li>Update any queries that use multi-value dimensions (MVDs) to use UNNEST in conjunction with other functions so that no MVD behavior is being relied upon. Type-aware schema discovery generates ARRAY typed columns instead of MVDs, so queries that use any MVD features will fail.</li><li>Be aware of mixed typed inputs and test how type-aware schema discovery handles them. Druid attempts to cast them as the least restrictive type.</li><li>If you notice issues with numeric types, you may need to explicitly cast them. Generally, Druid handles the coercion for you.</li><li>Update your dimension exclusion list and add any nested columns if you want to continue to exclude them. String-based schema discovery automatically ignores nested columns, but type-aware schema discovery will ingest them.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="including-the-same-column-as-a-dimension-and-a-metric">Including the same column as a dimension and a metric<a href="#including-the-same-column-as-a-dimension-and-a-metric" class="hash-link" aria-label="Direct link to Including the same column as a dimension and a metric" title="Direct link to Including the same column as a dimension and a metric"></a></h3><p>One workflow with unique IDs is to be able to filter on a particular ID, while still being able to do fast unique counts on the ID column. |
| If you are not using schema-less dimensions, this use case is supported by setting the <code>name</code> of the metric to something different than the dimension. |
| If you are using schema-less dimensions, the best practice here is to include the same column twice, once as a dimension, and as a <code>hyperUnique</code> metric. This may involve |
| some work at ETL time.</p><p>As an example, for schema-less dimensions, repeat the same column:</p><div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"> </span><span class="token property">"device_id_dim"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">123</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token property">"device_id_met"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">123</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>and in your <code>metricsSpec</code>, include:</p><div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#bfc7d5;--prism-background-color:#292d3e"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#bfc7d5"><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"> </span><span class="token property">"type"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"hyperUnique"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token property">"name"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"devices"</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token property">"fieldName"</span><span class="token operator" style="color:rgb(137, 221, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">"device_id_met"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><code>device_id_dim</code> should automatically get picked up as a dimension.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/docs/26.0.0/ingestion/ingestion-spec"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Ingestion spec</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/docs/26.0.0/development/extensions-core/kafka-ingestion"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Apache Kafka ingestion</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#druids-data-model" class="table-of-contents__link toc-highlight">Druid's data model</a></li><li><a href="#if-youre-coming-from-a" class="table-of-contents__link toc-highlight">If you're coming from a</a><ul><li><a href="#relational-model" class="table-of-contents__link toc-highlight">Relational model</a></li><li><a href="#time-series-model" class="table-of-contents__link toc-highlight">Time series model</a></li><li><a href="#log-aggregation-model" class="table-of-contents__link toc-highlight">Log aggregation model</a></li></ul></li><li><a href="#general-tips-and-best-practices" class="table-of-contents__link toc-highlight">General tips and best practices</a><ul><li><a href="#rollup" class="table-of-contents__link toc-highlight">Rollup</a></li><li><a href="#partitioning-and-sorting" class="table-of-contents__link toc-highlight">Partitioning and sorting</a></li><li><a href="#sketches-for-high-cardinality-columns" class="table-of-contents__link toc-highlight">Sketches for high cardinality columns</a></li><li><a href="#string-vs-numeric-dimensions" class="table-of-contents__link toc-highlight">String vs numeric dimensions</a></li><li><a href="#secondary-timestamps" class="table-of-contents__link toc-highlight">Secondary timestamps</a></li><li><a href="#nested-dimensions" class="table-of-contents__link toc-highlight">Nested dimensions</a></li><li><a href="#counting-the-number-of-ingested-events" class="table-of-contents__link toc-highlight">Counting the number of ingested events</a></li><li><a href="#schema-auto-discovery-for-dimensions" class="table-of-contents__link toc-highlight">Schema auto-discovery for dimensions</a></li><li><a href="#including-the-same-column-as-a-dimension-and-a-metric" class="table-of-contents__link toc-highlight">Including the same column as a dimension and a metric</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer"><div class="container container-fluid"><div class="footer__bottom text--center"><div class="margin-bottom--sm"><img src="/img/favicon.png" class="themedImage_ToTc themedImage--light_HNdA footer__logo"><img src="/img/favicon.png" class="themedImage_ToTc themedImage--dark_i4oU footer__logo"></div><div class="footer__copyright">Copyright © 2023 Apache Software Foundation. Except where otherwise noted, licensed under CC BY-SA 4.0. Apache Druid, Druid, and the Druid logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</div></div></div></footer></div> |
| <script src="/assets/js/runtime~main.38900cbf.js"></script> |
| <script src="/assets/js/main.5e106d68.js"></script> |
| </body> |
| </html> |
