<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Tutorial: Writing an ingestion spec · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/0.16.1-incubating/tutorials/tutorial-ingestion-spec.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="&lt;!--"/><meta name="docsearch:language" content="en"/><meta name="docsearch:version" content="0.16.1-incubating" /><meta property="og:title" content="Tutorial: Writing an ingestion spec · Apache Druid"/><meta property="og:type" content="website"/><meta property="og:url" content="https://druid.apache.org/index.html"/><meta property="og:description" content="&lt;!--"/><meta property="og:image" content="https://druid.apache.org/img/druid_nav.png"/><meta name="twitter:card" content="summary"/><meta name="twitter:image" content="https://druid.apache.org/img/druid_nav.png"/><link rel="shortcut icon" href="/img/favicon.png"/><link rel="stylesheet" href="https://cdn.jsdelivr.net/docsearch.js/1/docsearch.min.css"/><link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/default.min.css"/><script async="" src="https://www.googletagmanager.com/gtag/js?id=UA-131010415-1"></script><script>
              window.dataLayer = window.dataLayer || [];
              function gtag(){dataLayer.push(arguments); }
              gtag('js', new Date());
              gtag('config', 'UA-131010415-1');
            </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/druid_nav.png" alt="Apache Druid"/></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class=""><a href="/technology" target="_self">Technology</a></li><li class=""><a href="/use-cases" target="_self">Use Cases</a></li><li class=""><a href="/druid-powered" target="_self">Powered By</a></li><li class="siteNavGroupActive"><a href="/docs/0.16.1-incubating/design/index.html" target="_self">Docs</a></li><li class=""><a href="/community/" target="_self">Community</a></li><li class=""><a href="https://www.apache.org" target="_self">Apache</a></li><li class=""><a href="/downloads.html" target="_self">Download</a></li><li class="navSearchWrapper reactNavSearchWrapper"><input type="text" id="search_input_react" placeholder="Search" title="Search"/></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Tutorials</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle collapsible">Getting started<span class="arrow"><svg width="24" height="24" viewBox="0 0 24 24"><path fill="#565656" d="M7.41 15.41L12 10.83l4.59 4.58L18 14l-6-6-6 6z"></path><path d="M0 0h24v24H0z" fill="none"></path></svg></span></h3><ul class="hide"><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/design/index.html">Introduction to Apache Druid</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/tutorials/index.html">Quickstart</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/operations/single-server.html">Single server deployment</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/tutorials/cluster.html">Clustered deployment</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle collapsible">Tutorials<span class="arrow"><svg width="24" height="24" viewBox="0 0 24 24"><path fill="#565656" d="M7.41 15.41L12 10.83l4.59 4.58L18 14l-6-6-6 6z"></path><path d="M0 0h24v24H0z" fill="none"></path></svg></span></h3><ul class="hide"><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/tutorials/tutorial-batch.html">Loading files natively</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/tutorials/tutorial-kafka.html">Load from Apache Kafka</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/tutorials/tutorial-batch-hadoop.html">Load from Apache Hadoop</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/tutorials/tutorial-query.html">Querying data</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/tutorials/tutorial-rollup.html">Roll-up</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/tutorials/tutorial-retention.html">Configuring data retention</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/tutorials/tutorial-update-data.html">Updating existing data</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/tutorials/tutorial-compaction.html">Compacting segments</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/tutorials/tutorial-delete-data.html">Deleting data</a></li><li class="navListItem navListItemActive"><a class="navItem" href="/docs/0.16.1-incubating/tutorials/tutorial-ingestion-spec.html">Writing an ingestion spec</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/tutorials/tutorial-transform-spec.html">Transforming input data</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/tutorials/tutorial-kerberos-hadoop.html">Kerberized HDFS deep storage</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle collapsible">Design<span class="arrow"><svg width="24" height="24" viewBox="0 0 24 24"><path fill="#565656" d="M7.41 15.41L12 10.83l4.59 4.58L18 14l-6-6-6 6z"></path><path d="M0 0h24v24H0z" fill="none"></path></svg></span></h3><ul class="hide"><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/design/architecture.html">Design</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/design/segments.html">Segments</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/design/processes.html">Processes and servers</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/dependencies/deep-storage.html">Deep storage</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/dependencies/metadata-storage.html">Metadata storage</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/dependencies/zookeeper.html">ZooKeeper</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle collapsible">Data ingestion<span class="arrow"><svg width="24" height="24" viewBox="0 0 24 24"><path fill="#565656" d="M7.41 15.41L12 10.83l4.59 4.58L18 14l-6-6-6 6z"></path><path d="M0 0h24v24H0z" fill="none"></path></svg></span></h3><ul class="hide"><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/ingestion/index.html">Ingestion</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/ingestion/data-formats.html">Data formats</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/ingestion/schema-design.html">Schema design tips</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/ingestion/data-management.html">Data management</a></li><div class="navGroup subNavGroup"><h4 class="navGroupSubcategoryTitle">Stream ingestion</h4><ul><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-core/kafka-ingestion.html">Apache Kafka</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-core/kinesis-ingestion.html">Amazon Kinesis</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/ingestion/tranquility.html">Tranquility</a></li></ul></div><div class="navGroup subNavGroup"><h4 class="navGroupSubcategoryTitle">Batch ingestion</h4><ul><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/ingestion/native-batch.html">Native batch</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/ingestion/hadoop.html">Hadoop-based</a></li></ul></div><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/ingestion/tasks.html">Task reference</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/ingestion/faq.html">Troubleshooting FAQ</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle collapsible">Querying<span class="arrow"><svg width="24" height="24" viewBox="0 0 24 24"><path fill="#565656" d="M7.41 15.41L12 10.83l4.59 4.58L18 14l-6-6-6 6z"></path><path d="M0 0h24v24H0z" fill="none"></path></svg></span></h3><ul class="hide"><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/querying/sql.html">Druid SQL</a></li><div class="navGroup subNavGroup"><h4 class="navGroupSubcategoryTitle">Native query types</h4><ul><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/querying/querying.html">Making native queries</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/querying/timeseriesquery.html">Timeseries</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/querying/topnquery.html">TopN</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/querying/groupbyquery.html">GroupBy</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/querying/scan-query.html">Scan</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/querying/timeboundaryquery.html">TimeBoundary</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/querying/segmentmetadataquery.html">SegmentMetadata</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/querying/datasourcemetadataquery.html">DatasourceMetadata</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/querying/searchquery.html">Search</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/querying/select-query.html">Select</a></li></ul></div><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/querying/multi-value-dimensions.html">Multi-value dimensions</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/querying/lookups.html">Lookups</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/querying/joins.html">Joins</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/querying/multitenancy.html">Multitenancy considerations</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/querying/caching.html">Query caching</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/geo.html">Spatial filters</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle collapsible">Configuration<span class="arrow"><svg width="24" height="24" viewBox="0 0 24 24"><path fill="#565656" d="M7.41 15.41L12 10.83l4.59 4.58L18 14l-6-6-6 6z"></path><path d="M0 0h24v24H0z" fill="none"></path></svg></span></h3><ul class="hide"><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/configuration/index.html">Configuration reference</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions.html">Extensions</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/configuration/logging.html">Logging</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle collapsible">Operations<span class="arrow"><svg width="24" height="24" viewBox="0 0 24 24"><path fill="#565656" d="M7.41 15.41L12 10.83l4.59 4.58L18 14l-6-6-6 6z"></path><path d="M0 0h24v24H0z" fill="none"></path></svg></span></h3><ul class="hide"><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/operations/management-uis.html">Management UIs</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/operations/basic-cluster-tuning.html">Basic cluster tuning</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/operations/api-reference.html">API reference</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/operations/high-availability.html">High availability</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/operations/rolling-updates.html">Rolling updates</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/operations/rule-configuration.html">Retaining or automatically dropping data</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/operations/metrics.html">Metrics</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/operations/alerts.html">Alerts</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/operations/other-hadoop.html">Working with different versions of Apache Hadoop</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/operations/http-compression.html">HTTP compression</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/operations/recommendations.html">Recommendations</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/operations/tls-support.html">TLS support</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/operations/password-provider.html">Password providers</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/operations/dump-segment.html">dump-segment tool</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/operations/reset-cluster.html">reset-cluster tool</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/operations/insert-segment-to-db.html">insert-segment-to-db tool</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/operations/pull-deps.html">pull-deps tool</a></li><div class="navGroup subNavGroup"><h4 class="navGroupSubcategoryTitle">Misc</h4><ul><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/operations/deep-storage-migration.html">Deep storage migration</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/operations/druid-console.html">Web console</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/operations/export-metadata.html">Export Metadata Tool</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/operations/getting-started.html">Getting started with Apache Druid</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/operations/metadata-migration.html">Metadata Migration</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/operations/segment-optimization.html">Segment Size Optimization</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/operations/use_sbt_to_build_fat_jar.html">Content for build.sbt</a></li></ul></div></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle collapsible">Development<span class="arrow"><svg width="24" height="24" viewBox="0 0 24 24"><path fill="#565656" d="M7.41 15.41L12 10.83l4.59 4.58L18 14l-6-6-6 6z"></path><path d="M0 0h24v24H0z" fill="none"></path></svg></span></h3><ul class="hide"><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/overview.html">Developing on Druid</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/modules.html">Creating extensions</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/javascript.html">JavaScript functionality</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/build.html">Build from source</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/versioning.html">Versioning</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/experimental.html">Experimental features</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle collapsible">Misc<span class="arrow"><svg width="24" height="24" viewBox="0 0 24 24"><path fill="#565656" d="M7.41 15.41L12 10.83l4.59 4.58L18 14l-6-6-6 6z"></path><path d="M0 0h24v24H0z" fill="none"></path></svg></span></h3><ul class="hide"><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/misc/math-expr.html">Expressions</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/misc/papers-and-talks.html">Papers</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle collapsible">Hidden<span class="arrow"><svg width="24" height="24" viewBox="0 0 24 24"><path fill="#565656" d="M7.41 15.41L12 10.83l4.59 4.58L18 14l-6-6-6 6z"></path><path d="M0 0h24v24H0z" fill="none"></path></svg></span></h3><ul class="hide"><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/comparisons/druid-vs-elasticsearch.html">Apache Druid vs Elasticsearch</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/comparisons/druid-vs-key-value.html">Apache Druid vs. Key/Value Stores (HBase/Cassandra/OpenTSDB)</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/comparisons/druid-vs-kudu.html">Apache Druid vs Kudu</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/comparisons/druid-vs-redshift.html">Apache Druid vs Redshift</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/comparisons/druid-vs-spark.html">Apache Druid vs Spark</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/comparisons/druid-vs-sql-on-hadoop.html">Apache Druid vs SQL-on-Hadoop</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/design/auth.html">Authentication and Authorization</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/design/broker.html">Broker</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/design/coordinator.html">Coordinator Process</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/design/historical.html">Historical Process</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/design/indexer.html">Indexer Process</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/design/indexing-service.html">Indexing Service</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/design/middlemanager.html">MiddleManager Process</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/design/overlord.html">Overlord Process</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/design/router.html">Router Process</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/design/peons.html">Peons</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-core/approximate-histograms.html">Approximate Histogram aggregators</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-core/avro.html">Apache Avro</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-core/bloom-filter.html">Bloom Filter</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-core/datasketches-extension.html">DataSketches extension</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-core/datasketches-hll.html">DataSketches HLL Sketch module</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-core/datasketches-quantiles.html">DataSketches Quantiles Sketch module</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-core/datasketches-theta.html">DataSketches Theta Sketch module</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-core/datasketches-tuple.html">DataSketches Tuple Sketch module</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-core/druid-basic-security.html">Basic Security</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-core/druid-kerberos.html">Kerberos</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-core/druid-lookups.html">Cached Lookup Module</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-core/google.html">Google Cloud Storage</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-core/hdfs.html">HDFS</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-core/kafka-extraction-namespace.html">Apache Kafka Lookups</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-core/lookups-cached-global.html">Globally Cached Lookups</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-core/mysql.html">MySQL Metadata Store</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-core/orc.html">ORC Extension</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-core/parquet.html">Apache Parquet Extension</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-core/postgresql.html">PostgreSQL Metadata Store</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-core/protobuf.html">Protobuf</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-core/s3.html">S3-compatible</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-core/simple-client-sslcontext.html">Simple SSLContext Provider Module</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-core/stats.html">Stats aggregator</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-core/test-stats.html">Test Stats Aggregators</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-contrib/ambari-metrics-emitter.html">Ambari Metrics Emitter</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-contrib/azure.html">Microsoft Azure</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-contrib/cassandra.html">Apache Cassandra</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-contrib/cloudfiles.html">Rackspace Cloud Files</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-contrib/distinctcount.html">DistinctCount Aggregator</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-contrib/graphite.html">Graphite Emitter</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/querying/aggregations.html">Aggregations</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/querying/datasource.html">Datasources</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/querying/dimensionspecs.html">Transforming Dimension Values</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/querying/filters.html">Query Filters</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/querying/granularities.html">Aggregation Granularity</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/querying/having.html">Filter groupBy query results</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/querying/hll-old.html">Cardinality/HyperUnique aggregators</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/querying/limitspec.html">Sort groupBy query results</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/querying/post-aggregations.html">Post-Aggregations</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/querying/query-context.html">Query context</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/querying/searchqueryspec.html">Refining search queries</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/querying/sorting-orders.html">Sorting Orders</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/querying/topnmetricspec.html">TopNMetricSpec</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/querying/virtual-columns.html">Virtual Columns</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-contrib/influx.html">InfluxDB Line Protocol Parser</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-contrib/influxdb-emitter.html">InfluxDB Emitter</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-contrib/kafka-emitter.html">Kafka Emitter</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-contrib/materialized-view.html">Materialized View</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-contrib/momentsketch-quantiles.html">Moment Sketches for Approximate Quantiles module</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-contrib/moving-average-query.html">development/extensions-contrib/moving-average-query</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-contrib/opentsdb-emitter.html">OpenTSDB Emitter</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-contrib/redis-cache.html">Druid Redis Cache</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-contrib/sqlserver.html">Microsoft SQLServer</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-contrib/statsd.html">StatsD Emitter</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-contrib/tdigestsketch-quantiles.html">T-Digest Quantiles Sketch module</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-contrib/thrift.html">Thrift</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/development/extensions-contrib/time-min-max.html">Timestamp Min/Max aggregators</a></li><li class="navListItem"><a class="navItem" href="/docs/0.16.1-incubating/ingestion/standalone-realtime.html">Realtime Process</a></li></ul></div></div></section></div><script>
            var coll = document.getElementsByClassName('collapsible');
            var checkActiveCategory = true;
            for (var i = 0; i < coll.length; i++) {
              var links = coll[i].nextElementSibling.getElementsByTagName('*');
              if (checkActiveCategory){
                for (var j = 0; j < links.length; j++) {
                  if (links[j].classList.contains('navListItemActive')){
                    coll[i].nextElementSibling.classList.toggle('hide');
                    coll[i].childNodes[1].classList.toggle('rotate');
                    checkActiveCategory = false;
                    break;
                  }
                }
              }

              coll[i].addEventListener('click', function() {
                var arrow = this.childNodes[1];
                arrow.classList.toggle('rotate');
                var content = this.nextElementSibling;
                content.classList.toggle('hide');
              });
            }

            document.addEventListener('DOMContentLoaded', function() {
              createToggler('#navToggler', '#docsNav', 'docsSliderActive');
              createToggler('#tocToggler', 'body', 'tocActive');

              var headings = document.querySelector('.toc-headings');
              headings && headings.addEventListener('click', function(event) {
                var el = event.target;
                while(el !== headings){
                  if (el.tagName === 'A') {
                    document.body.classList.remove('tocActive');
                    break;
                  } else{
                    el = el.parentNode;
                  }
                }
              }, false);

              function createToggler(togglerSelector, targetSelector, className) {
                var toggler = document.querySelector(togglerSelector);
                var target = document.querySelector(targetSelector);

                if (!toggler) {
                  return;
                }

                toggler.onclick = function(event) {
                  event.preventDefault();

                  target.classList.toggle(className);
                };
              }
            });
        </script></nav></div><div class="container mainContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/incubator-druid/edit/master/docs/tutorials/tutorial-ingestion-spec.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 class="postHeaderTitle">Tutorial: Writing an ingestion spec</h1></header><article><div><span><!--
  ~ Licensed to the Apache Software Foundation (ASF) under one
  ~ or more contributor license agreements.  See the NOTICE file
  ~ distributed with this work for additional information
  ~ regarding copyright ownership.  The ASF licenses this file
  ~ to you under the Apache License, Version 2.0 (the
  ~ "License"); you may not use this file except in compliance
  ~ with the License.  You may obtain a copy of the License at
  ~
  ~   http://www.apache.org/licenses/LICENSE-2.0
  ~
  ~ Unless required by applicable law or agreed to in writing,
  ~ software distributed under the License is distributed on an
  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  ~ KIND, either express or implied.  See the License for the
  ~ specific language governing permissions and limitations
  ~ under the License.
  -->
<p>This tutorial will guide the reader through the process of defining an ingestion spec, pointing out key considerations and guidelines.</p>
<p>For this tutorial, we'll assume you've already downloaded Apache Druid (incubating) as described in
the <a href="index.html">single-machine quickstart</a> and have it running on your local machine.</p>
<p>It will also be helpful to have finished <a href="/docs/0.16.1-incubating/tutorials/tutorial-batch.html">Tutorial: Loading a file</a>, <a href="/docs/0.16.1-incubating/tutorials/tutorial-query.html">Tutorial: Querying data</a>, and <a href="/docs/0.16.1-incubating/tutorials/tutorial-rollup.html">Tutorial: Rollup</a>.</p>
<h2><a class="anchor" aria-hidden="true" id="example-data"></a><a href="#example-data" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Example data</h2>
<p>Suppose we have the following network flow data:</p>
<ul>
<li><code>srcIP</code>: IP address of sender</li>
<li><code>srcPort</code>: Port of sender</li>
<li><code>dstIP</code>: IP address of receiver</li>
<li><code>dstPort</code>: Port of receiver</li>
<li><code>protocol</code>: IP protocol number</li>
<li><code>packets</code>: number of packets transmitted</li>
<li><code>bytes</code>: number of bytes transmitted</li>
<li><code>cost</code>: the cost of sending the traffic</li>
</ul>
<pre><code class="hljs css language-json">{<span class="hljs-attr">"ts"</span>:<span class="hljs-string">"2018-01-01T01:01:35Z"</span>,<span class="hljs-attr">"srcIP"</span>:<span class="hljs-string">"1.1.1.1"</span>, <span class="hljs-attr">"dstIP"</span>:<span class="hljs-string">"2.2.2.2"</span>, <span class="hljs-attr">"srcPort"</span>:<span class="hljs-number">2000</span>, <span class="hljs-attr">"dstPort"</span>:<span class="hljs-number">3000</span>, <span class="hljs-attr">"protocol"</span>: <span class="hljs-number">6</span>, <span class="hljs-attr">"packets"</span>:<span class="hljs-number">10</span>, <span class="hljs-attr">"bytes"</span>:<span class="hljs-number">1000</span>, <span class="hljs-attr">"cost"</span>: <span class="hljs-number">1.4</span>}
{<span class="hljs-attr">"ts"</span>:<span class="hljs-string">"2018-01-01T01:01:51Z"</span>,<span class="hljs-attr">"srcIP"</span>:<span class="hljs-string">"1.1.1.1"</span>, <span class="hljs-attr">"dstIP"</span>:<span class="hljs-string">"2.2.2.2"</span>, <span class="hljs-attr">"srcPort"</span>:<span class="hljs-number">2000</span>, <span class="hljs-attr">"dstPort"</span>:<span class="hljs-number">3000</span>, <span class="hljs-attr">"protocol"</span>: <span class="hljs-number">6</span>, <span class="hljs-attr">"packets"</span>:<span class="hljs-number">20</span>, <span class="hljs-attr">"bytes"</span>:<span class="hljs-number">2000</span>, <span class="hljs-attr">"cost"</span>: <span class="hljs-number">3.1</span>}
{<span class="hljs-attr">"ts"</span>:<span class="hljs-string">"2018-01-01T01:01:59Z"</span>,<span class="hljs-attr">"srcIP"</span>:<span class="hljs-string">"1.1.1.1"</span>, <span class="hljs-attr">"dstIP"</span>:<span class="hljs-string">"2.2.2.2"</span>, <span class="hljs-attr">"srcPort"</span>:<span class="hljs-number">2000</span>, <span class="hljs-attr">"dstPort"</span>:<span class="hljs-number">3000</span>, <span class="hljs-attr">"protocol"</span>: <span class="hljs-number">6</span>, <span class="hljs-attr">"packets"</span>:<span class="hljs-number">30</span>, <span class="hljs-attr">"bytes"</span>:<span class="hljs-number">3000</span>, <span class="hljs-attr">"cost"</span>: <span class="hljs-number">0.4</span>}
{<span class="hljs-attr">"ts"</span>:<span class="hljs-string">"2018-01-01T01:02:14Z"</span>,<span class="hljs-attr">"srcIP"</span>:<span class="hljs-string">"1.1.1.1"</span>, <span class="hljs-attr">"dstIP"</span>:<span class="hljs-string">"2.2.2.2"</span>, <span class="hljs-attr">"srcPort"</span>:<span class="hljs-number">5000</span>, <span class="hljs-attr">"dstPort"</span>:<span class="hljs-number">7000</span>, <span class="hljs-attr">"protocol"</span>: <span class="hljs-number">6</span>, <span class="hljs-attr">"packets"</span>:<span class="hljs-number">40</span>, <span class="hljs-attr">"bytes"</span>:<span class="hljs-number">4000</span>, <span class="hljs-attr">"cost"</span>: <span class="hljs-number">7.9</span>}
{<span class="hljs-attr">"ts"</span>:<span class="hljs-string">"2018-01-01T01:02:29Z"</span>,<span class="hljs-attr">"srcIP"</span>:<span class="hljs-string">"1.1.1.1"</span>, <span class="hljs-attr">"dstIP"</span>:<span class="hljs-string">"2.2.2.2"</span>, <span class="hljs-attr">"srcPort"</span>:<span class="hljs-number">5000</span>, <span class="hljs-attr">"dstPort"</span>:<span class="hljs-number">7000</span>, <span class="hljs-attr">"protocol"</span>: <span class="hljs-number">6</span>, <span class="hljs-attr">"packets"</span>:<span class="hljs-number">50</span>, <span class="hljs-attr">"bytes"</span>:<span class="hljs-number">5000</span>, <span class="hljs-attr">"cost"</span>: <span class="hljs-number">10.2</span>}
{<span class="hljs-attr">"ts"</span>:<span class="hljs-string">"2018-01-01T01:03:29Z"</span>,<span class="hljs-attr">"srcIP"</span>:<span class="hljs-string">"1.1.1.1"</span>, <span class="hljs-attr">"dstIP"</span>:<span class="hljs-string">"2.2.2.2"</span>, <span class="hljs-attr">"srcPort"</span>:<span class="hljs-number">5000</span>, <span class="hljs-attr">"dstPort"</span>:<span class="hljs-number">7000</span>, <span class="hljs-attr">"protocol"</span>: <span class="hljs-number">6</span>, <span class="hljs-attr">"packets"</span>:<span class="hljs-number">60</span>, <span class="hljs-attr">"bytes"</span>:<span class="hljs-number">6000</span>, <span class="hljs-attr">"cost"</span>: <span class="hljs-number">4.3</span>}
{<span class="hljs-attr">"ts"</span>:<span class="hljs-string">"2018-01-01T02:33:14Z"</span>,<span class="hljs-attr">"srcIP"</span>:<span class="hljs-string">"7.7.7.7"</span>, <span class="hljs-attr">"dstIP"</span>:<span class="hljs-string">"8.8.8.8"</span>, <span class="hljs-attr">"srcPort"</span>:<span class="hljs-number">4000</span>, <span class="hljs-attr">"dstPort"</span>:<span class="hljs-number">5000</span>, <span class="hljs-attr">"protocol"</span>: <span class="hljs-number">17</span>, <span class="hljs-attr">"packets"</span>:<span class="hljs-number">100</span>, <span class="hljs-attr">"bytes"</span>:<span class="hljs-number">10000</span>, <span class="hljs-attr">"cost"</span>: <span class="hljs-number">22.4</span>}
{<span class="hljs-attr">"ts"</span>:<span class="hljs-string">"2018-01-01T02:33:45Z"</span>,<span class="hljs-attr">"srcIP"</span>:<span class="hljs-string">"7.7.7.7"</span>, <span class="hljs-attr">"dstIP"</span>:<span class="hljs-string">"8.8.8.8"</span>, <span class="hljs-attr">"srcPort"</span>:<span class="hljs-number">4000</span>, <span class="hljs-attr">"dstPort"</span>:<span class="hljs-number">5000</span>, <span class="hljs-attr">"protocol"</span>: <span class="hljs-number">17</span>, <span class="hljs-attr">"packets"</span>:<span class="hljs-number">200</span>, <span class="hljs-attr">"bytes"</span>:<span class="hljs-number">20000</span>, <span class="hljs-attr">"cost"</span>: <span class="hljs-number">34.5</span>}
{<span class="hljs-attr">"ts"</span>:<span class="hljs-string">"2018-01-01T02:35:45Z"</span>,<span class="hljs-attr">"srcIP"</span>:<span class="hljs-string">"7.7.7.7"</span>, <span class="hljs-attr">"dstIP"</span>:<span class="hljs-string">"8.8.8.8"</span>, <span class="hljs-attr">"srcPort"</span>:<span class="hljs-number">4000</span>, <span class="hljs-attr">"dstPort"</span>:<span class="hljs-number">5000</span>, <span class="hljs-attr">"protocol"</span>: <span class="hljs-number">17</span>, <span class="hljs-attr">"packets"</span>:<span class="hljs-number">300</span>, <span class="hljs-attr">"bytes"</span>:<span class="hljs-number">30000</span>, <span class="hljs-attr">"cost"</span>: <span class="hljs-number">46.3</span>}
</code></pre>
<p>Save the JSON contents above into a file called <code>ingestion-tutorial-data.json</code> in <code>quickstart/</code>.</p>
<p>Let's walk through the process of defining an ingestion spec that can load this data.</p>
<p>For this tutorial, we will be using the native batch indexing task. When using other task types, some aspects of the ingestion spec will differ, and this tutorial will point out such areas.</p>
<h2><a class="anchor" aria-hidden="true" id="defining-the-schema"></a><a href="#defining-the-schema" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Defining the schema</h2>
<p>The core element of a Druid ingestion spec is the <code>dataSchema</code>. The <code>dataSchema</code> defines how to parse input data into a set of columns that will be stored in Druid.</p>
<p>Let's start with an empty <code>dataSchema</code> and add fields to it as we progress through the tutorial.</p>
<p>Create a new file called <code>ingestion-tutorial-index.json</code> in <code>quickstart/</code> with the following contents:</p>
<pre><code class="hljs css language-json">"dataSchema" : {}
</code></pre>
<p>We will be making successive edits to this ingestion spec as we progress through the tutorial.</p>
<h3><a class="anchor" aria-hidden="true" id="datasource-name"></a><a href="#datasource-name" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Datasource name</h3>
<p>The datasource name is specified by the <code>dataSource</code> parameter in the <code>dataSchema</code>.</p>
<pre><code class="hljs css language-json">"dataSchema" : {
  "dataSource" : "ingestion-tutorial",
}
</code></pre>
<p>Let's call the tutorial datasource <code>ingestion-tutorial</code>.</p>
<h3><a class="anchor" aria-hidden="true" id="choose-a-parser"></a><a href="#choose-a-parser" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Choose a parser</h3>
<p>A <code>dataSchema</code> has a <code>parser</code> field, which defines the parser that Druid will use to interpret the input data.</p>
<p>Since our input data is represented as JSON strings, we'll use a <code>string</code> parser with <code>json</code> format:</p>
<pre><code class="hljs css language-json">"dataSchema" : {
  "dataSource" : "ingestion-tutorial",
  "parser" : {
    "type" : "string",
    "parseSpec" : {
      "format" : "json"
    }
  }
}
</code></pre>
<h3><a class="anchor" aria-hidden="true" id="time-column"></a><a href="#time-column" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Time column</h3>
<p>The <code>parser</code> needs to know how to extract the main timestamp field from the input data. When using a <code>json</code> type <code>parseSpec</code>, the timestamp is defined in a <code>timestampSpec</code>.</p>
<p>The timestamp column in our input data is named &quot;ts&quot;, containing ISO 8601 timestamps, so let's add a <code>timestampSpec</code> with that information to the <code>parseSpec</code>:</p>
<pre><code class="hljs css language-json">"dataSchema" : {
  "dataSource" : "ingestion-tutorial",
  "parser" : {
    "type" : "string",
    "parseSpec" : {
      "format" : "json",
      "timestampSpec" : {
        "format" : "iso",
        "column" : "ts"
      }
    }
  }
}
</code></pre>
<h3><a class="anchor" aria-hidden="true" id="column-types"></a><a href="#column-types" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Column types</h3>
<p>Now that we've defined the time column, let's look at definitions for other columns.</p>
<p>Druid supports the following column types: String, Long, Float, Double. We will see how these are used in the following sections.</p>
<p>Before we move on to how we define our other non-time columns, let's discuss <code>rollup</code> first.</p>
<h3><a class="anchor" aria-hidden="true" id="rollup"></a><a href="#rollup" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Rollup</h3>
<p>When ingesting data, we must consider whether we wish to use rollup or not.</p>
<ul>
<li><p>If rollup is enabled, we will need to separate the input columns into two categories, &quot;dimensions&quot; and &quot;metrics&quot;. &quot;Dimensions&quot; are the grouping columns for rollup, while &quot;metrics&quot; are the columns that will be aggregated.</p></li>
<li><p>If rollup is disabled, then all columns are treated as &quot;dimensions&quot; and no pre-aggregation occurs.</p></li>
</ul>
<p>For this tutorial, let's enable rollup. This is specified with a <code>granularitySpec</code> on the <code>dataSchema</code>.</p>
<p>Note that the <code>granularitySpec</code> lies outside of the <code>parser</code>. We will revisit the <code>parser</code> soon when we define our dimensions and metrics.</p>
<pre><code class="hljs css language-json">"dataSchema" : {
  "dataSource" : "ingestion-tutorial",
  "parser" : {
    "type" : "string",
    "parseSpec" : {
      "format" : "json",
      "timestampSpec" : {
        "format" : "iso",
        "column" : "ts"
      }
    }
  },
  "granularitySpec" : {
    "rollup" : true
  }
}

</code></pre>
<h4><a class="anchor" aria-hidden="true" id="choosing-dimensions-and-metrics"></a><a href="#choosing-dimensions-and-metrics" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Choosing dimensions and metrics</h4>
<p>For this example dataset, the following is a sensible split for &quot;dimensions&quot; and &quot;metrics&quot;:</p>
<ul>
<li>Dimensions: srcIP, srcPort, dstIP, dstPort, protocol</li>
<li>Metrics: packets, bytes, cost</li>
</ul>
<p>The dimensions here are a group of properties that identify a unidirectional flow of IP traffic, while the metrics represent facts about the IP traffic flow specified by a dimension grouping.</p>
<p>Let's look at how to define these dimensions and metrics within the ingestion spec.</p>
<h4><a class="anchor" aria-hidden="true" id="dimensions"></a><a href="#dimensions" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Dimensions</h4>
<p>Dimensions are specified with a <code>dimensionsSpec</code> inside the <code>parseSpec</code>.</p>
<pre><code class="hljs css language-json">"dataSchema" : {
  "dataSource" : "ingestion-tutorial",
  "parser" : {
    "type" : "string",
    "parseSpec" : {
      "format" : "json",
      "timestampSpec" : {
        "format" : "iso",
        "column" : "ts"
      },
      "dimensionsSpec" : {
        "dimensions": [
          "srcIP",
          { "name" : "srcPort", "type" : "long" },
          { "name" : "dstIP", "type" : "string" },
          { "name" : "dstPort", "type" : "long" },
          { "name" : "protocol", "type" : "string" }
        ]
      }
    }
  },
  "granularitySpec" : {
    "rollup" : true
  }
}
</code></pre>
<p>Each dimension has a <code>name</code> and a <code>type</code>, where <code>type</code> can be &quot;long&quot;, &quot;float&quot;, &quot;double&quot;, or &quot;string&quot;.</p>
<p>Note that <code>srcIP</code> is a &quot;string&quot; dimension; for string dimensions, it is enough to specify just a dimension name, since &quot;string&quot; is the default dimension type.</p>
<p>Also note that <code>protocol</code> is a numeric value in the input data, but we are ingesting it as a &quot;string&quot; column; Druid will coerce the input longs to strings during ingestion.</p>
<h5><a class="anchor" aria-hidden="true" id="strings-vs-numerics"></a><a href="#strings-vs-numerics" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Strings vs. Numerics</h5>
<p>Should a numeric input be ingested as a numeric dimension or as a string dimension?</p>
<p>Numeric dimensions have the following pros/cons relative to String dimensions:</p>
<ul>
<li>Pros: Numeric representation can result in smaller column sizes on disk and lower processing overhead when reading values from the column</li>
<li>Cons: Numeric dimensions do not have indices, so filtering on them will often be slower than filtering on an equivalent String dimension (which has bitmap indices)</li>
</ul>
<h4><a class="anchor" aria-hidden="true" id="metrics"></a><a href="#metrics" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Metrics</h4>
<p>Metrics are specified with a <code>metricsSpec</code> inside the <code>dataSchema</code>:</p>
<pre><code class="hljs css language-json">"dataSchema" : {
  "dataSource" : "ingestion-tutorial",
  "parser" : {
    "type" : "string",
    "parseSpec" : {
      "format" : "json",
      "timestampSpec" : {
        "format" : "iso",
        "column" : "ts"
      },
      "dimensionsSpec" : {
        "dimensions": [
          "srcIP",
          { "name" : "srcPort", "type" : "long" },
          { "name" : "dstIP", "type" : "string" },
          { "name" : "dstPort", "type" : "long" },
          { "name" : "protocol", "type" : "string" }
        ]
      }
    }
  },
  "metricsSpec" : [
    { "type" : "count", "name" : "count" },
    { "type" : "longSum", "name" : "packets", "fieldName" : "packets" },
    { "type" : "longSum", "name" : "bytes", "fieldName" : "bytes" },
    { "type" : "doubleSum", "name" : "cost", "fieldName" : "cost" }
  ],
  "granularitySpec" : {
    "rollup" : true
  }
}
</code></pre>
<p>When defining a metric, it is necessary to specify what type of aggregation should be performed on that column during rollup.</p>
<p>Here we have defined long sum aggregations on the two long metric columns, <code>packets</code> and <code>bytes</code>, and a double sum aggregation for the <code>cost</code> column.</p>
<p>Note that the <code>metricsSpec</code> is on a different nesting level than <code>dimensionSpec</code> or <code>parseSpec</code>; it belongs on the same nesting level as <code>parser</code> within the <code>dataSchema</code>.</p>
<p>Note that we have also defined a <code>count</code> aggregator. The count aggregator will track how many rows in the original input data contributed to a &quot;rolled up&quot; row in the final ingested data.</p>
<h3><a class="anchor" aria-hidden="true" id="no-rollup"></a><a href="#no-rollup" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>No rollup</h3>
<p>If we were not using rollup, all columns would be specified in the <code>dimensionsSpec</code>, e.g.:</p>
<pre><code class="hljs css language-json">      "dimensionsSpec" : {
        "dimensions": [
          "srcIP",
          { "name" : "srcPort", "type" : "long" },
          { "name" : "dstIP", "type" : "string" },
          { "name" : "dstPort", "type" : "long" },
          { "name" : "protocol", "type" : "string" },
          { "name" : "packets", "type" : "long" },
          { "name" : "bytes", "type" : "long" },
          { "name" : "srcPort", "type" : "double" }
        ]
      },
</code></pre>
<h3><a class="anchor" aria-hidden="true" id="define-granularities"></a><a href="#define-granularities" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Define granularities</h3>
<p>At this point, we are done defining the <code>parser</code> and <code>metricsSpec</code> within the <code>dataSchema</code> and we are almost done writing the ingestion spec.</p>
<p>There are some additional properties we need to set in the <code>granularitySpec</code>:</p>
<ul>
<li>Type of granularitySpec: <code>uniform</code> and <code>arbitrary</code> are the two supported types. For this tutorial, we will use a <code>uniform</code> granularity spec, where all segments have uniform interval sizes (for example, all segments cover an hour's worth of data).</li>
<li>The segment granularity: what size of time interval should a single segment contain data for? e.g., <code>DAY</code>, <code>WEEK</code></li>
<li>The bucketing granularity of the timestamps in the time column (referred to as <code>queryGranularity</code>)</li>
</ul>
<h4><a class="anchor" aria-hidden="true" id="segment-granularity"></a><a href="#segment-granularity" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Segment granularity</h4>
<p>Segment granularity is configured by the <code>segmentGranularity</code> property in the <code>granularitySpec</code>. For this tutorial, we'll create hourly segments:</p>
<pre><code class="hljs css language-json">"dataSchema" : {
  "dataSource" : "ingestion-tutorial",
  "parser" : {
    "type" : "string",
    "parseSpec" : {
      "format" : "json",
      "timestampSpec" : {
        "format" : "iso",
        "column" : "ts"
      },
      "dimensionsSpec" : {
        "dimensions": [
          "srcIP",
          { "name" : "srcPort", "type" : "long" },
          { "name" : "dstIP", "type" : "string" },
          { "name" : "dstPort", "type" : "long" },
          { "name" : "protocol", "type" : "string" }
        ]
      }
    }
  },
  "metricsSpec" : [
    { "type" : "count", "name" : "count" },
    { "type" : "longSum", "name" : "packets", "fieldName" : "packets" },
    { "type" : "longSum", "name" : "bytes", "fieldName" : "bytes" },
    { "type" : "doubleSum", "name" : "cost", "fieldName" : "cost" }
  ],
  "granularitySpec" : {
    "type" : "uniform",
    "segmentGranularity" : "HOUR",
    "rollup" : true
  }
}
</code></pre>
<p>Our input data has events from two separate hours, so this task will generate two segments.</p>
<h4><a class="anchor" aria-hidden="true" id="query-granularity"></a><a href="#query-granularity" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Query granularity</h4>
<p>The query granularity is configured by the <code>queryGranularity</code> property in the <code>granularitySpec</code>. For this tutorial, let's use minute granularity:</p>
<pre><code class="hljs css language-json">"dataSchema" : {
  "dataSource" : "ingestion-tutorial",
  "parser" : {
    "type" : "string",
    "parseSpec" : {
      "format" : "json",
      "timestampSpec" : {
        "format" : "iso",
        "column" : "ts"
      },
      "dimensionsSpec" : {
        "dimensions": [
          "srcIP",
          { "name" : "srcPort", "type" : "long" },
          { "name" : "dstIP", "type" : "string" },
          { "name" : "dstPort", "type" : "long" },
          { "name" : "protocol", "type" : "string" }
        ]
      }
    }
  },
  "metricsSpec" : [
    { "type" : "count", "name" : "count" },
    { "type" : "longSum", "name" : "packets", "fieldName" : "packets" },
    { "type" : "longSum", "name" : "bytes", "fieldName" : "bytes" },
    { "type" : "doubleSum", "name" : "cost", "fieldName" : "cost" }
  ],
  "granularitySpec" : {
    "type" : "uniform",
    "segmentGranularity" : "HOUR",
    "queryGranularity" : "MINUTE"
    "rollup" : true
  }
}
</code></pre>
<p>To see the effect of the query granularity, let's look at this row from the raw input data:</p>
<pre><code class="hljs css language-json">{<span class="hljs-attr">"ts"</span>:<span class="hljs-string">"2018-01-01T01:03:29Z"</span>,<span class="hljs-attr">"srcIP"</span>:<span class="hljs-string">"1.1.1.1"</span>, <span class="hljs-attr">"dstIP"</span>:<span class="hljs-string">"2.2.2.2"</span>, <span class="hljs-attr">"srcPort"</span>:<span class="hljs-number">5000</span>, <span class="hljs-attr">"dstPort"</span>:<span class="hljs-number">7000</span>, <span class="hljs-attr">"protocol"</span>: <span class="hljs-number">6</span>, <span class="hljs-attr">"packets"</span>:<span class="hljs-number">60</span>, <span class="hljs-attr">"bytes"</span>:<span class="hljs-number">6000</span>, <span class="hljs-attr">"cost"</span>: <span class="hljs-number">4.3</span>}
</code></pre>
<p>When this row is ingested with minute queryGranularity, Druid will floor the row's timestamp to minute buckets:</p>
<pre><code class="hljs css language-json">{<span class="hljs-attr">"ts"</span>:<span class="hljs-string">"2018-01-01T01:03:00Z"</span>,<span class="hljs-attr">"srcIP"</span>:<span class="hljs-string">"1.1.1.1"</span>, <span class="hljs-attr">"dstIP"</span>:<span class="hljs-string">"2.2.2.2"</span>, <span class="hljs-attr">"srcPort"</span>:<span class="hljs-number">5000</span>, <span class="hljs-attr">"dstPort"</span>:<span class="hljs-number">7000</span>, <span class="hljs-attr">"protocol"</span>: <span class="hljs-number">6</span>, <span class="hljs-attr">"packets"</span>:<span class="hljs-number">60</span>, <span class="hljs-attr">"bytes"</span>:<span class="hljs-number">6000</span>, <span class="hljs-attr">"cost"</span>: <span class="hljs-number">4.3</span>}
</code></pre>
<h4><a class="anchor" aria-hidden="true" id="define-an-interval-batch-only"></a><a href="#define-an-interval-batch-only" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Define an interval (batch only)</h4>
<p>For batch tasks, it is necessary to define a time interval. Input rows with timestamps outside of the time interval will not be ingested.</p>
<p>The interval is also specified in the <code>granularitySpec</code>:</p>
<pre><code class="hljs css language-json">"dataSchema" : {
  "dataSource" : "ingestion-tutorial",
  "parser" : {
    "type" : "string",
    "parseSpec" : {
      "format" : "json",
      "timestampSpec" : {
        "format" : "iso",
        "column" : "ts"
      },
      "dimensionsSpec" : {
        "dimensions": [
          "srcIP",
          { "name" : "srcPort", "type" : "long" },
          { "name" : "dstIP", "type" : "string" },
          { "name" : "dstPort", "type" : "long" },
          { "name" : "protocol", "type" : "string" }
        ]
      }
    }
  },
  "metricsSpec" : [
    { "type" : "count", "name" : "count" },
    { "type" : "longSum", "name" : "packets", "fieldName" : "packets" },
    { "type" : "longSum", "name" : "bytes", "fieldName" : "bytes" },
    { "type" : "doubleSum", "name" : "cost", "fieldName" : "cost" }
  ],
  "granularitySpec" : {
    "type" : "uniform",
    "segmentGranularity" : "HOUR",
    "queryGranularity" : "MINUTE",
    "intervals" : ["2018-01-01/2018-01-02"],
    "rollup" : true
  }
}
</code></pre>
<h2><a class="anchor" aria-hidden="true" id="define-the-task-type"></a><a href="#define-the-task-type" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Define the task type</h2>
<p>We've now finished defining our <code>dataSchema</code>. The remaining steps are to place the <code>dataSchema</code> we created into an ingestion task spec, and specify the input source.</p>
<p>The <code>dataSchema</code> is shared across all task types, but each task type has its own specification format. For this tutorial, we will use the native batch ingestion task:</p>
<pre><code class="hljs css language-json">{
  <span class="hljs-attr">"type"</span> : <span class="hljs-string">"index"</span>,
  <span class="hljs-attr">"spec"</span> : {
    <span class="hljs-attr">"dataSchema"</span> : {
      <span class="hljs-attr">"dataSource"</span> : <span class="hljs-string">"ingestion-tutorial"</span>,
      <span class="hljs-attr">"parser"</span> : {
        <span class="hljs-attr">"type"</span> : <span class="hljs-string">"string"</span>,
        <span class="hljs-attr">"parseSpec"</span> : {
          <span class="hljs-attr">"format"</span> : <span class="hljs-string">"json"</span>,
          <span class="hljs-attr">"timestampSpec"</span> : {
            <span class="hljs-attr">"format"</span> : <span class="hljs-string">"iso"</span>,
            <span class="hljs-attr">"column"</span> : <span class="hljs-string">"ts"</span>
          },
          <span class="hljs-attr">"dimensionsSpec"</span> : {
            <span class="hljs-attr">"dimensions"</span>: [
              <span class="hljs-string">"srcIP"</span>,
              { <span class="hljs-attr">"name"</span> : <span class="hljs-string">"srcPort"</span>, <span class="hljs-attr">"type"</span> : <span class="hljs-string">"long"</span> },
              { <span class="hljs-attr">"name"</span> : <span class="hljs-string">"dstIP"</span>, <span class="hljs-attr">"type"</span> : <span class="hljs-string">"string"</span> },
              { <span class="hljs-attr">"name"</span> : <span class="hljs-string">"dstPort"</span>, <span class="hljs-attr">"type"</span> : <span class="hljs-string">"long"</span> },
              { <span class="hljs-attr">"name"</span> : <span class="hljs-string">"protocol"</span>, <span class="hljs-attr">"type"</span> : <span class="hljs-string">"string"</span> }
            ]
          }
        }
      },
      <span class="hljs-attr">"metricsSpec"</span> : [
        { <span class="hljs-attr">"type"</span> : <span class="hljs-string">"count"</span>, <span class="hljs-attr">"name"</span> : <span class="hljs-string">"count"</span> },
        { <span class="hljs-attr">"type"</span> : <span class="hljs-string">"longSum"</span>, <span class="hljs-attr">"name"</span> : <span class="hljs-string">"packets"</span>, <span class="hljs-attr">"fieldName"</span> : <span class="hljs-string">"packets"</span> },
        { <span class="hljs-attr">"type"</span> : <span class="hljs-string">"longSum"</span>, <span class="hljs-attr">"name"</span> : <span class="hljs-string">"bytes"</span>, <span class="hljs-attr">"fieldName"</span> : <span class="hljs-string">"bytes"</span> },
        { <span class="hljs-attr">"type"</span> : <span class="hljs-string">"doubleSum"</span>, <span class="hljs-attr">"name"</span> : <span class="hljs-string">"cost"</span>, <span class="hljs-attr">"fieldName"</span> : <span class="hljs-string">"cost"</span> }
      ],
      <span class="hljs-attr">"granularitySpec"</span> : {
        <span class="hljs-attr">"type"</span> : <span class="hljs-string">"uniform"</span>,
        <span class="hljs-attr">"segmentGranularity"</span> : <span class="hljs-string">"HOUR"</span>,
        <span class="hljs-attr">"queryGranularity"</span> : <span class="hljs-string">"MINUTE"</span>,
        <span class="hljs-attr">"intervals"</span> : [<span class="hljs-string">"2018-01-01/2018-01-02"</span>],
        <span class="hljs-attr">"rollup"</span> : <span class="hljs-literal">true</span>
      }
    }
  }
}
</code></pre>
<h2><a class="anchor" aria-hidden="true" id="define-the-input-source"></a><a href="#define-the-input-source" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Define the input source</h2>
<p>Now let's define our input source, which is specified in an <code>ioConfig</code> object. Each task type has its own type of <code>ioConfig</code>. The native batch task uses &quot;firehoses&quot; to read input data, so let's configure a &quot;local&quot; firehose to read the example netflow data we saved earlier:</p>
<pre><code class="hljs css language-json">    "ioConfig" : {
      "type" : "index",
      "firehose" : {
        "type" : "local",
        "baseDir" : "quickstart/",
        "filter" : "ingestion-tutorial-data.json"
      }
    }
</code></pre>
<pre><code class="hljs css language-json">{
  <span class="hljs-attr">"type"</span> : <span class="hljs-string">"index"</span>,
  <span class="hljs-attr">"spec"</span> : {
    <span class="hljs-attr">"dataSchema"</span> : {
      <span class="hljs-attr">"dataSource"</span> : <span class="hljs-string">"ingestion-tutorial"</span>,
      <span class="hljs-attr">"parser"</span> : {
        <span class="hljs-attr">"type"</span> : <span class="hljs-string">"string"</span>,
        <span class="hljs-attr">"parseSpec"</span> : {
          <span class="hljs-attr">"format"</span> : <span class="hljs-string">"json"</span>,
          <span class="hljs-attr">"timestampSpec"</span> : {
            <span class="hljs-attr">"format"</span> : <span class="hljs-string">"iso"</span>,
            <span class="hljs-attr">"column"</span> : <span class="hljs-string">"ts"</span>
          },
          <span class="hljs-attr">"dimensionsSpec"</span> : {
            <span class="hljs-attr">"dimensions"</span>: [
              <span class="hljs-string">"srcIP"</span>,
              { <span class="hljs-attr">"name"</span> : <span class="hljs-string">"srcPort"</span>, <span class="hljs-attr">"type"</span> : <span class="hljs-string">"long"</span> },
              { <span class="hljs-attr">"name"</span> : <span class="hljs-string">"dstIP"</span>, <span class="hljs-attr">"type"</span> : <span class="hljs-string">"string"</span> },
              { <span class="hljs-attr">"name"</span> : <span class="hljs-string">"dstPort"</span>, <span class="hljs-attr">"type"</span> : <span class="hljs-string">"long"</span> },
              { <span class="hljs-attr">"name"</span> : <span class="hljs-string">"protocol"</span>, <span class="hljs-attr">"type"</span> : <span class="hljs-string">"string"</span> }
            ]
          }
        }
      },
      <span class="hljs-attr">"metricsSpec"</span> : [
        { <span class="hljs-attr">"type"</span> : <span class="hljs-string">"count"</span>, <span class="hljs-attr">"name"</span> : <span class="hljs-string">"count"</span> },
        { <span class="hljs-attr">"type"</span> : <span class="hljs-string">"longSum"</span>, <span class="hljs-attr">"name"</span> : <span class="hljs-string">"packets"</span>, <span class="hljs-attr">"fieldName"</span> : <span class="hljs-string">"packets"</span> },
        { <span class="hljs-attr">"type"</span> : <span class="hljs-string">"longSum"</span>, <span class="hljs-attr">"name"</span> : <span class="hljs-string">"bytes"</span>, <span class="hljs-attr">"fieldName"</span> : <span class="hljs-string">"bytes"</span> },
        { <span class="hljs-attr">"type"</span> : <span class="hljs-string">"doubleSum"</span>, <span class="hljs-attr">"name"</span> : <span class="hljs-string">"cost"</span>, <span class="hljs-attr">"fieldName"</span> : <span class="hljs-string">"cost"</span> }
      ],
      <span class="hljs-attr">"granularitySpec"</span> : {
        <span class="hljs-attr">"type"</span> : <span class="hljs-string">"uniform"</span>,
        <span class="hljs-attr">"segmentGranularity"</span> : <span class="hljs-string">"HOUR"</span>,
        <span class="hljs-attr">"queryGranularity"</span> : <span class="hljs-string">"MINUTE"</span>,
        <span class="hljs-attr">"intervals"</span> : [<span class="hljs-string">"2018-01-01/2018-01-02"</span>],
        <span class="hljs-attr">"rollup"</span> : <span class="hljs-literal">true</span>
      }
    },
    <span class="hljs-attr">"ioConfig"</span> : {
      <span class="hljs-attr">"type"</span> : <span class="hljs-string">"index"</span>,
      <span class="hljs-attr">"firehose"</span> : {
        <span class="hljs-attr">"type"</span> : <span class="hljs-string">"local"</span>,
        <span class="hljs-attr">"baseDir"</span> : <span class="hljs-string">"quickstart/"</span>,
        <span class="hljs-attr">"filter"</span> : <span class="hljs-string">"ingestion-tutorial-data.json"</span>
      }
    }
  }
}
</code></pre>
<h2><a class="anchor" aria-hidden="true" id="additional-tuning"></a><a href="#additional-tuning" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Additional tuning</h2>
<p>Each ingestion task has a <code>tuningConfig</code> section that allows users to tune various ingestion parameters.</p>
<p>As an example, let's add a <code>tuningConfig</code> that sets a target segment size for the native batch ingestion task:</p>
<pre><code class="hljs css language-json">    "tuningConfig" : {
      "type" : "index",
      "maxRowsPerSegment" : 5000000
    }
</code></pre>
<p>Note that each ingestion task has its own type of <code>tuningConfig</code>.</p>
<h2><a class="anchor" aria-hidden="true" id="final-spec"></a><a href="#final-spec" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Final spec</h2>
<p>We've finished defining the ingestion spec, it should now look like the following:</p>
<pre><code class="hljs css language-json">{
  <span class="hljs-attr">"type"</span> : <span class="hljs-string">"index"</span>,
  <span class="hljs-attr">"spec"</span> : {
    <span class="hljs-attr">"dataSchema"</span> : {
      <span class="hljs-attr">"dataSource"</span> : <span class="hljs-string">"ingestion-tutorial"</span>,
      <span class="hljs-attr">"parser"</span> : {
        <span class="hljs-attr">"type"</span> : <span class="hljs-string">"string"</span>,
        <span class="hljs-attr">"parseSpec"</span> : {
          <span class="hljs-attr">"format"</span> : <span class="hljs-string">"json"</span>,
          <span class="hljs-attr">"timestampSpec"</span> : {
            <span class="hljs-attr">"format"</span> : <span class="hljs-string">"iso"</span>,
            <span class="hljs-attr">"column"</span> : <span class="hljs-string">"ts"</span>
          },
          <span class="hljs-attr">"dimensionsSpec"</span> : {
            <span class="hljs-attr">"dimensions"</span>: [
              <span class="hljs-string">"srcIP"</span>,
              { <span class="hljs-attr">"name"</span> : <span class="hljs-string">"srcPort"</span>, <span class="hljs-attr">"type"</span> : <span class="hljs-string">"long"</span> },
              { <span class="hljs-attr">"name"</span> : <span class="hljs-string">"dstIP"</span>, <span class="hljs-attr">"type"</span> : <span class="hljs-string">"string"</span> },
              { <span class="hljs-attr">"name"</span> : <span class="hljs-string">"dstPort"</span>, <span class="hljs-attr">"type"</span> : <span class="hljs-string">"long"</span> },
              { <span class="hljs-attr">"name"</span> : <span class="hljs-string">"protocol"</span>, <span class="hljs-attr">"type"</span> : <span class="hljs-string">"string"</span> }
            ]
          }
        }
      },
      <span class="hljs-attr">"metricsSpec"</span> : [
        { <span class="hljs-attr">"type"</span> : <span class="hljs-string">"count"</span>, <span class="hljs-attr">"name"</span> : <span class="hljs-string">"count"</span> },
        { <span class="hljs-attr">"type"</span> : <span class="hljs-string">"longSum"</span>, <span class="hljs-attr">"name"</span> : <span class="hljs-string">"packets"</span>, <span class="hljs-attr">"fieldName"</span> : <span class="hljs-string">"packets"</span> },
        { <span class="hljs-attr">"type"</span> : <span class="hljs-string">"longSum"</span>, <span class="hljs-attr">"name"</span> : <span class="hljs-string">"bytes"</span>, <span class="hljs-attr">"fieldName"</span> : <span class="hljs-string">"bytes"</span> },
        { <span class="hljs-attr">"type"</span> : <span class="hljs-string">"doubleSum"</span>, <span class="hljs-attr">"name"</span> : <span class="hljs-string">"cost"</span>, <span class="hljs-attr">"fieldName"</span> : <span class="hljs-string">"cost"</span> }
      ],
      <span class="hljs-attr">"granularitySpec"</span> : {
        <span class="hljs-attr">"type"</span> : <span class="hljs-string">"uniform"</span>,
        <span class="hljs-attr">"segmentGranularity"</span> : <span class="hljs-string">"HOUR"</span>,
        <span class="hljs-attr">"queryGranularity"</span> : <span class="hljs-string">"MINUTE"</span>,
        <span class="hljs-attr">"intervals"</span> : [<span class="hljs-string">"2018-01-01/2018-01-02"</span>],
        <span class="hljs-attr">"rollup"</span> : <span class="hljs-literal">true</span>
      }
    },
    <span class="hljs-attr">"ioConfig"</span> : {
      <span class="hljs-attr">"type"</span> : <span class="hljs-string">"index"</span>,
      <span class="hljs-attr">"firehose"</span> : {
        <span class="hljs-attr">"type"</span> : <span class="hljs-string">"local"</span>,
        <span class="hljs-attr">"baseDir"</span> : <span class="hljs-string">"quickstart/"</span>,
        <span class="hljs-attr">"filter"</span> : <span class="hljs-string">"ingestion-tutorial-data.json"</span>
      }
    },
    <span class="hljs-attr">"tuningConfig"</span> : {
      <span class="hljs-attr">"type"</span> : <span class="hljs-string">"index"</span>,
      <span class="hljs-attr">"maxRowsPerSegment"</span> : <span class="hljs-number">5000000</span>
    }
  }
}
</code></pre>
<h2><a class="anchor" aria-hidden="true" id="submit-the-task-and-query-the-data"></a><a href="#submit-the-task-and-query-the-data" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Submit the task and query the data</h2>
<p>From the apache-druid-0.16.1-incubating package root, run the following command:</p>
<pre><code class="hljs css language-bash">bin/post-index-task --file quickstart/ingestion-tutorial-index.json --url http://localhost:8081
</code></pre>
<p>After the script completes, we will query the data.</p>
<p>Let's run <code>bin/dsql</code> and issue a <code>select * from &quot;ingestion-tutorial&quot;;</code> query to see what data was ingested.</p>
<pre><code class="hljs css language-bash">$ bin/dsql
Welcome to dsql, the <span class="hljs-built_in">command</span>-line client <span class="hljs-keyword">for</span> Druid SQL.
Type <span class="hljs-string">"\h"</span> <span class="hljs-keyword">for</span> <span class="hljs-built_in">help</span>.
dsql&gt; select * from <span class="hljs-string">"ingestion-tutorial"</span>;

┌──────────────────────────┬───────┬──────┬───────┬─────────┬─────────┬─────────┬──────────┬─────────┬─────────┐
│ __time                   │ bytes │ cost │ count │ dstIP   │ dstPort │ packets │ protocol │ srcIP   │ srcPort │
├──────────────────────────┼───────┼──────┼───────┼─────────┼─────────┼─────────┼──────────┼─────────┼─────────┤
│ 2018-01-01T01:01:00.000Z │  6000 │  4.9 │     3 │ 2.2.2.2 │    3000 │      60 │ 6        │ 1.1.1.1 │    2000 │
│ 2018-01-01T01:02:00.000Z │  9000 │ 18.1 │     2 │ 2.2.2.2 │    7000 │      90 │ 6        │ 1.1.1.1 │    5000 │
│ 2018-01-01T01:03:00.000Z │  6000 │  4.3 │     1 │ 2.2.2.2 │    7000 │      60 │ 6        │ 1.1.1.1 │    5000 │
│ 2018-01-01T02:33:00.000Z │ 30000 │ 56.9 │     2 │ 8.8.8.8 │    5000 │     300 │ 17       │ 7.7.7.7 │    4000 │
│ 2018-01-01T02:35:00.000Z │ 30000 │ 46.3 │     1 │ 8.8.8.8 │    5000 │     300 │ 17       │ 7.7.7.7 │    4000 │
└──────────────────────────┴───────┴──────┴───────┴─────────┴─────────┴─────────┴──────────┴─────────┴─────────┘
Retrieved 5 rows <span class="hljs-keyword">in</span> 0.12s.

dsql&gt;
</code></pre>
</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/0.16.1-incubating/tutorials/tutorial-delete-data.html"><span class="arrow-prev">← </span><span>Deleting data</span></a><a class="docs-next button" href="/docs/0.16.1-incubating/tutorials/tutorial-transform-spec.html"><span>Transforming input data</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#example-data">Example data</a></li><li><a href="#defining-the-schema">Defining the schema</a><ul class="toc-headings"><li><a href="#datasource-name">Datasource name</a></li><li><a href="#choose-a-parser">Choose a parser</a></li><li><a href="#time-column">Time column</a></li><li><a href="#column-types">Column types</a></li><li><a href="#rollup">Rollup</a></li><li><a href="#no-rollup">No rollup</a></li><li><a href="#define-granularities">Define granularities</a></li></ul></li><li><a href="#define-the-task-type">Define the task type</a></li><li><a href="#define-the-input-source">Define the input source</a></li><li><a href="#additional-tuning">Additional tuning</a></li><li><a href="#final-spec">Final spec</a></li><li><a href="#submit-the-task-and-query-the-data">Submit the task and query the data</a></li></ul></nav></div><footer class="nav-footer druid-footer" id="footer"><div class="container"><div class="text-center"><p><a href="/technology">Technology</a> · <a href="/use-cases">Use Cases</a> · <a href="/druid-powered">Powered by Druid</a> · <a href="/docs/0.16.1-incubating/latest">Docs</a> · <a href="/community/">Community</a> · <a href="/downloads.html">Download</a> · <a href="/faq">FAQ</a></p></div><div class="text-center"><a title="Join the user group" href="https://groups.google.com/forum/#!forum/druid-user" target="_blank"><span class="fa fa-comments"></span></a> · <a title="Follow Druid" href="https://twitter.com/druidio" target="_blank"><span class="fab fa-twitter"></span></a> · <a title="Download via Apache" href="https://www.apache.org/dyn/closer.cgi?path=/incubator/druid/{{ site.druid_versions[0].versions[0].version }}/apache-druid-{{ site.druid_versions[0].versions[0].version }}-bin.tar.gz" target="_blank"><span class="fas fa-feather"></span></a> · <a title="GitHub" href="https://github.com/apache/incubator-druid" target="_blank"><span class="fab fa-github"></span></a></div><div class="text-center license">Copyright © 2019 <a href="https://www.apache.org/" target="_blank">Apache Software Foundation</a>.<br/>Except where otherwise noted, licensed under <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">CC BY-SA 4.0</a>.<br/>Apache Druid, Druid, and the Druid logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</div></div></footer></div><script type="text/javascript" src="https://cdn.jsdelivr.net/docsearch.js/1/docsearch.min.js"></script><script>
                document.addEventListener('keyup', function(e) {
                  if (e.target !== document.body) {
                    return;
                  }
                  // keyCode for '/' (slash)
                  if (e.keyCode === 191) {
                    const search = document.getElementById('search_input_react');
                    search && search.focus();
                  }
                });
              </script><script>
              var search = docsearch({
                
                apiKey: '2de99082a9f38e49dfaa059bbe4c901d',
                indexName: 'apache_druid',
                inputSelector: '#search_input_react',
                algoliaOptions: {"facetFilters":["language:en","version:0.16.1-incubating"]}
              });
            </script></body></html>