| <!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Task reference · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/0.23.0/ingestion/tasks.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="<!--"/><meta name="docsearch:language" content="en"/><meta name="docsearch:version" content="0.23.0" /><meta property="og:title" content="Task reference · Apache Druid"/><meta property="og:type" content="website"/><meta property="og:url" content="https://druid.apache.org/index.html"/><meta property="og:description" content="<!--"/><meta property="og:image" content="https://druid.apache.org/img/druid_nav.png"/><meta name="twitter:card" content="summary"/><meta name="twitter:image" content="https://druid.apache.org/img/druid_nav.png"/><link rel="shortcut icon" href="/img/favicon.png"/><link rel="stylesheet" href="https://cdn.jsdelivr.net/docsearch.js/1/docsearch.min.css"/><link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/default.min.css"/><script async="" src="https://www.googletagmanager.com/gtag/js?id=UA-131010415-1"></script><script> |
| window.dataLayer = window.dataLayer || []; |
| function gtag(){dataLayer.push(arguments); } |
| gtag('js', new Date()); |
| gtag('config', 'UA-131010415-1'); |
| </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/druid_nav.png" alt="Apache Druid"/></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class=""><a href="/technology" target="_self">Technology</a></li><li class=""><a href="/use-cases" target="_self">Use Cases</a></li><li class=""><a href="/druid-powered" target="_self">Powered By</a></li><li class="siteNavGroupActive"><a href="/docs/0.23.0/design/index.html" target="_self">Docs</a></li><li class=""><a href="/community/" target="_self">Community</a></li><li class=""><a href="https://www.apache.org" target="_self">Apache</a></li><li class=""><a href="/downloads.html" target="_self">Download</a></li><li class="navSearchWrapper reactNavSearchWrapper"><input type="text" id="search_input_react" placeholder="Search" title="Search"/></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Ingestion</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle collapsible">Getting started<span class="arrow"><svg width="24" height="24" viewBox="0 0 24 24"><path fill="#565656" d="M7.41 15.41L12 10.83l4.59 4.58L18 14l-6-6-6 6z"></path><path d="M0 0h24v24H0z" fill="none"></path></svg></span></h3><ul class="hide"><li class="navListItem"><a class="navItem" href="/docs/0.23.0/design/index.html">Introduction to Apache Druid</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/tutorials/index.html">Quickstart</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/tutorials/docker.html">Docker</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/single-server.html">Single server deployment</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/tutorials/cluster.html">Clustered deployment</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle collapsible">Tutorials<span class="arrow"><svg width="24" height="24" viewBox="0 0 24 24"><path fill="#565656" d="M7.41 15.41L12 10.83l4.59 4.58L18 14l-6-6-6 6z"></path><path d="M0 0h24v24H0z" fill="none"></path></svg></span></h3><ul class="hide"><li class="navListItem"><a class="navItem" href="/docs/0.23.0/tutorials/tutorial-batch.html">Loading files natively</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/tutorials/tutorial-kafka.html">Load from Apache Kafka</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/tutorials/tutorial-batch-hadoop.html">Load from Apache Hadoop</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/tutorials/tutorial-query.html">Querying data</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/tutorials/tutorial-rollup.html">Roll-up</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/tutorials/tutorial-retention.html">Configuring data retention</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/tutorials/tutorial-update-data.html">Updating existing data</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/tutorials/tutorial-compaction.html">Compacting segments</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/tutorials/tutorial-delete-data.html">Deleting data</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/tutorials/tutorial-ingestion-spec.html">Writing an ingestion spec</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/tutorials/tutorial-transform-spec.html">Transforming input data</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/tutorials/tutorial-kerberos-hadoop.html">Kerberized HDFS deep storage</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle collapsible">Design<span class="arrow"><svg width="24" height="24" viewBox="0 0 24 24"><path fill="#565656" d="M7.41 15.41L12 10.83l4.59 4.58L18 14l-6-6-6 6z"></path><path d="M0 0h24v24H0z" fill="none"></path></svg></span></h3><ul class="hide"><li class="navListItem"><a class="navItem" href="/docs/0.23.0/design/architecture.html">Design</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/design/segments.html">Segments</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/design/processes.html">Processes and servers</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/dependencies/deep-storage.html">Deep storage</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/dependencies/metadata-storage.html">Metadata storage</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/dependencies/zookeeper.html">ZooKeeper</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle collapsible">Ingestion<span class="arrow"><svg width="24" height="24" viewBox="0 0 24 24"><path fill="#565656" d="M7.41 15.41L12 10.83l4.59 4.58L18 14l-6-6-6 6z"></path><path d="M0 0h24v24H0z" fill="none"></path></svg></span></h3><ul class="hide"><li class="navListItem"><a class="navItem" href="/docs/0.23.0/ingestion/index.html">Ingestion</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/ingestion/data-formats.html">Data formats</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/ingestion/data-model.html">Data model</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/ingestion/rollup.html">Data rollup</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/ingestion/partitioning.html">Partitioning</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/ingestion/ingestion-spec.html">Ingestion spec</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/ingestion/schema-design.html">Schema design tips</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/ingestion/data-management.html">Data management</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/ingestion/compaction.html">Compaction</a></li><div class="navGroup subNavGroup"><h4 class="navGroupSubcategoryTitle">Stream ingestion</h4><ul><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/kafka-ingestion.html">Apache Kafka ingestion</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/kafka-supervisor-reference.html">Apache Kafka supervisor</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/kafka-supervisor-operations.html">Apache Kafka operations</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/kinesis-ingestion.html">Amazon Kinesis</a></li></ul></div><div class="navGroup subNavGroup"><h4 class="navGroupSubcategoryTitle">Batch ingestion</h4><ul><li class="navListItem"><a class="navItem" href="/docs/0.23.0/ingestion/native-batch.html">Native batch</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/ingestion/native-batch-simple-task.html">Simple task indexing</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/ingestion/native-batch-input-sources.html">Input sources</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/ingestion/native-batch-firehose.html">Firehose</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/ingestion/hadoop.html">Hadoop-based</a></li></ul></div><li class="navListItem navListItemActive"><a class="navItem" href="/docs/0.23.0/ingestion/tasks.html">Task reference</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/ingestion/faq.html">Troubleshooting FAQ</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle collapsible">Querying<span class="arrow"><svg width="24" height="24" viewBox="0 0 24 24"><path fill="#565656" d="M7.41 15.41L12 10.83l4.59 4.58L18 14l-6-6-6 6z"></path><path d="M0 0h24v24H0z" fill="none"></path></svg></span></h3><ul class="hide"><div class="navGroup subNavGroup"><h4 class="navGroupSubcategoryTitle">Druid SQL</h4><ul><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/sql.html">Overview and syntax</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/sql-data-types.html">SQL data types</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/sql-operators.html">Operators</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/sql-scalar.html">Scalar functions</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/sql-aggregations.html">Aggregation functions</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/sql-multivalue-string-functions.html">Multi-value string functions</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/sql-api.html">Druid SQL API</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/sql-jdbc.html">JDBC driver API</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/sql-query-context.html">SQL query context</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/sql-metadata-tables.html">SQL metadata tables</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/sql-translation.html">SQL query translation</a></li></ul></div><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/querying.html">Native queries</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/query-execution.html">Query execution</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/troubleshooting.html">Troubleshooting</a></li><div class="navGroup subNavGroup"><h4 class="navGroupSubcategoryTitle">Concepts</h4><ul><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/datasource.html">Datasources</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/joins.html">Joins</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/lookups.html">Lookups</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/multi-value-dimensions.html">Multi-value dimensions</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/multitenancy.html">Multitenancy</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/caching.html">Query caching</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/using-caching.html">Using query caching</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/query-context.html">Query context</a></li></ul></div><div class="navGroup subNavGroup"><h4 class="navGroupSubcategoryTitle">Native query types</h4><ul><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/timeseriesquery.html">Timeseries</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/topnquery.html">TopN</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/groupbyquery.html">GroupBy</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/scan-query.html">Scan</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/searchquery.html">Search</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/timeboundaryquery.html">TimeBoundary</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/segmentmetadataquery.html">SegmentMetadata</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/datasourcemetadataquery.html">DatasourceMetadata</a></li></ul></div><div class="navGroup subNavGroup"><h4 class="navGroupSubcategoryTitle">Native query components</h4><ul><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/filters.html">Filters</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/granularities.html">Granularities</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/dimensionspecs.html">Dimensions</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/aggregations.html">Aggregations</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/post-aggregations.html">Post-aggregations</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/misc/math-expr.html">Expressions</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/having.html">Having filters (groupBy)</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/limitspec.html">Sorting and limiting (groupBy)</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/topnmetricspec.html">Sorting (topN)</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/sorting-orders.html">String comparators</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/virtual-columns.html">Virtual columns</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/geo.html">Spatial filters</a></li></ul></div></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle collapsible">Configuration<span class="arrow"><svg width="24" height="24" viewBox="0 0 24 24"><path fill="#565656" d="M7.41 15.41L12 10.83l4.59 4.58L18 14l-6-6-6 6z"></path><path d="M0 0h24v24H0z" fill="none"></path></svg></span></h3><ul class="hide"><li class="navListItem"><a class="navItem" href="/docs/0.23.0/configuration/index.html">Configuration reference</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions.html">Extensions</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/configuration/logging.html">Logging</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle collapsible">Operations<span class="arrow"><svg width="24" height="24" viewBox="0 0 24 24"><path fill="#565656" d="M7.41 15.41L12 10.83l4.59 4.58L18 14l-6-6-6 6z"></path><path d="M0 0h24v24H0z" fill="none"></path></svg></span></h3><ul class="hide"><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/druid-console.html">Web console</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/getting-started.html">Getting started with Apache Druid</a></li><div class="navGroup subNavGroup"><h4 class="navGroupSubcategoryTitle">Security</h4><ul><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/security-overview.html">Security overview</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/security-user-auth.html">User authentication and authorization</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/auth-ldap.html">LDAP auth</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/password-provider.html">Password providers</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/dynamic-config-provider.html">Dynamic Config Providers</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/tls-support.html">TLS support</a></li></ul></div><div class="navGroup subNavGroup"><h4 class="navGroupSubcategoryTitle">Performance tuning</h4><ul><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/basic-cluster-tuning.html">Basic cluster tuning</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/segment-optimization.html">Segment Size Optimization</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/mixed-workloads.html">Mixed workloads</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/http-compression.html">HTTP compression</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/clean-metadata-store.html">Automated metadata cleanup</a></li></ul></div><div class="navGroup subNavGroup"><h4 class="navGroupSubcategoryTitle">Monitoring</h4><ul><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/request-logging.html">Request logging</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/metrics.html">Metrics</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/alerts.html">Alerts</a></li></ul></div><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/api-reference.html">API reference</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/high-availability.html">High availability</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/rolling-updates.html">Rolling updates</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/rule-configuration.html">Retaining or automatically dropping data</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/other-hadoop.html">Working with different versions of Apache Hadoop</a></li><div class="navGroup subNavGroup"><h4 class="navGroupSubcategoryTitle">Misc</h4><ul><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/management-uis.html">Legacy Management UIs</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/dump-segment.html">dump-segment tool</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/reset-cluster.html">reset-cluster tool</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/insert-segment-to-db.html">insert-segment-to-db tool</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/pull-deps.html">pull-deps tool</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/deep-storage-migration.html">Deep storage migration</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/export-metadata.html">Export Metadata Tool</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/metadata-migration.html">Metadata Migration</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/use_sbt_to_build_fat_jar.html">Content for build.sbt</a></li></ul></div></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle collapsible">Development<span class="arrow"><svg width="24" height="24" viewBox="0 0 24 24"><path fill="#565656" d="M7.41 15.41L12 10.83l4.59 4.58L18 14l-6-6-6 6z"></path><path d="M0 0h24v24H0z" fill="none"></path></svg></span></h3><ul class="hide"><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/overview.html">Developing on Druid</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/modules.html">Creating extensions</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/javascript.html">JavaScript functionality</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/build.html">Build from source</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/versioning.html">Versioning</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/experimental.html">Experimental features</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle collapsible">Misc<span class="arrow"><svg width="24" height="24" viewBox="0 0 24 24"><path fill="#565656" d="M7.41 15.41L12 10.83l4.59 4.58L18 14l-6-6-6 6z"></path><path d="M0 0h24v24H0z" fill="none"></path></svg></span></h3><ul class="hide"><li class="navListItem"><a class="navItem" href="/docs/0.23.0/misc/papers-and-talks.html">Papers</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle collapsible">Hidden<span class="arrow"><svg width="24" height="24" viewBox="0 0 24 24"><path fill="#565656" d="M7.41 15.41L12 10.83l4.59 4.58L18 14l-6-6-6 6z"></path><path d="M0 0h24v24H0z" fill="none"></path></svg></span></h3><ul class="hide"><li class="navListItem"><a class="navItem" href="/docs/0.23.0/comparisons/druid-vs-elasticsearch.html">Apache Druid vs Elasticsearch</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/comparisons/druid-vs-key-value.html">Apache Druid vs. Key/Value Stores (HBase/Cassandra/OpenTSDB)</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/comparisons/druid-vs-kudu.html">Apache Druid vs Kudu</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/comparisons/druid-vs-redshift.html">Apache Druid vs Redshift</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/comparisons/druid-vs-spark.html">Apache Druid vs Spark</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/comparisons/druid-vs-sql-on-hadoop.html">Apache Druid vs SQL-on-Hadoop</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/design/auth.html">Authentication and Authorization</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/design/broker.html">Broker</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/design/coordinator.html">Coordinator Process</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/design/historical.html">Historical Process</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/design/indexer.html">Indexer Process</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/design/indexing-service.html">Indexing Service</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/design/middlemanager.html">MiddleManager Process</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/design/overlord.html">Overlord Process</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/design/router.html">Router Process</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/design/peons.html">Peons</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/approximate-histograms.html">Approximate Histogram aggregators</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/avro.html">Apache Avro</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/azure.html">Microsoft Azure</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/bloom-filter.html">Bloom Filter</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/datasketches-extension.html">DataSketches extension</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/datasketches-hll.html">DataSketches HLL Sketch module</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/datasketches-quantiles.html">DataSketches Quantiles Sketch module</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/datasketches-theta.html">DataSketches Theta Sketch module</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/datasketches-tuple.html">DataSketches Tuple Sketch module</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/druid-basic-security.html">Basic Security</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/druid-kerberos.html">Kerberos</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/druid-lookups.html">Cached Lookup Module</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/druid-ranger-security.html">Apache Ranger Security</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/google.html">Google Cloud Storage</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/hdfs.html">HDFS</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/kafka-extraction-namespace.html">Apache Kafka Lookups</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/lookups-cached-global.html">Globally Cached Lookups</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/mysql.html">MySQL Metadata Store</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/orc.html">ORC Extension</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/druid-pac4j.html">Druid pac4j based Security extension</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/parquet.html">Apache Parquet Extension</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/postgresql.html">PostgreSQL Metadata Store</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/protobuf.html">Protobuf</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/s3.html">S3-compatible</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/simple-client-sslcontext.html">Simple SSLContext Provider Module</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/stats.html">Stats aggregator</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/test-stats.html">Test Stats Aggregators</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/druid-aws-rds.html">Druid AWS RDS Module</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-core/kubernetes.html">Kubernetes</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-contrib/ambari-metrics-emitter.html">Ambari Metrics Emitter</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-contrib/cassandra.html">Apache Cassandra</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-contrib/cloudfiles.html">Rackspace Cloud Files</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-contrib/distinctcount.html">DistinctCount Aggregator</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-contrib/graphite.html">Graphite Emitter</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-contrib/influx.html">InfluxDB Line Protocol Parser</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-contrib/influxdb-emitter.html">InfluxDB Emitter</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-contrib/kafka-emitter.html">Kafka Emitter</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-contrib/materialized-view.html">Materialized View</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-contrib/momentsketch-quantiles.html">Moment Sketches for Approximate Quantiles module</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-contrib/moving-average-query.html">Moving Average Query</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-contrib/opentsdb-emitter.html">OpenTSDB Emitter</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-contrib/redis-cache.html">Druid Redis Cache</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-contrib/sqlserver.html">Microsoft SQLServer</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-contrib/statsd.html">StatsD Emitter</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-contrib/tdigestsketch-quantiles.html">T-Digest Quantiles Sketch module</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-contrib/thrift.html">Thrift</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-contrib/time-min-max.html">Timestamp Min/Max aggregators</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-contrib/gce-extensions.html">GCE Extensions</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-contrib/aliyun-oss.html">Aliyun OSS</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/development/extensions-contrib/prometheus.html">Prometheus Emitter</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/operations/kubernetes.html">kubernetes</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/hll-old.html">Cardinality/HyperUnique aggregators</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/querying/select-query.html">Select</a></li><li class="navListItem"><a class="navItem" href="/docs/0.23.0/ingestion/standalone-realtime.html">Realtime Process</a></li></ul></div></div></section></div><script> |
| var coll = document.getElementsByClassName('collapsible'); |
| var checkActiveCategory = true; |
| for (var i = 0; i < coll.length; i++) { |
| var links = coll[i].nextElementSibling.getElementsByTagName('*'); |
| if (checkActiveCategory){ |
| for (var j = 0; j < links.length; j++) { |
| if (links[j].classList.contains('navListItemActive')){ |
| coll[i].nextElementSibling.classList.toggle('hide'); |
| coll[i].childNodes[1].classList.toggle('rotate'); |
| checkActiveCategory = false; |
| break; |
| } |
| } |
| } |
| |
| coll[i].addEventListener('click', function() { |
| var arrow = this.childNodes[1]; |
| arrow.classList.toggle('rotate'); |
| var content = this.nextElementSibling; |
| content.classList.toggle('hide'); |
| }); |
| } |
| |
| document.addEventListener('DOMContentLoaded', function() { |
| createToggler('#navToggler', '#docsNav', 'docsSliderActive'); |
| createToggler('#tocToggler', 'body', 'tocActive'); |
| |
| var headings = document.querySelector('.toc-headings'); |
| headings && headings.addEventListener('click', function(event) { |
| var el = event.target; |
| while(el !== headings){ |
| if (el.tagName === 'A') { |
| document.body.classList.remove('tocActive'); |
| break; |
| } else{ |
| el = el.parentNode; |
| } |
| } |
| }, false); |
| |
| function createToggler(togglerSelector, targetSelector, className) { |
| var toggler = document.querySelector(togglerSelector); |
| var target = document.querySelector(targetSelector); |
| |
| if (!toggler) { |
| return; |
| } |
| |
| toggler.onclick = function(event) { |
| event.preventDefault(); |
| |
| target.classList.toggle(className); |
| }; |
| } |
| }); |
| </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/ingestion/tasks.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Task reference</h1></header><article><div><span><!-- |
| ~ Licensed to the Apache Software Foundation (ASF) under one |
| ~ or more contributor license agreements. See the NOTICE file |
| ~ distributed with this work for additional information |
| ~ regarding copyright ownership. The ASF licenses this file |
| ~ to you under the Apache License, Version 2.0 (the |
| ~ "License"); you may not use this file except in compliance |
| ~ with the License. You may obtain a copy of the License at |
| ~ |
| ~ http://www.apache.org/licenses/LICENSE-2.0 |
| ~ |
| ~ Unless required by applicable law or agreed to in writing, |
| ~ software distributed under the License is distributed on an |
| ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| ~ KIND, either express or implied. See the License for the |
| ~ specific language governing permissions and limitations |
| ~ under the License. |
| --> |
| <p>Tasks do all <a href="/docs/0.23.0/ingestion/index.html">ingestion</a>-related work in Druid.</p> |
| <p>For batch ingestion, you will generally submit tasks directly to Druid using the |
| <a href="/docs/0.23.0/operations/api-reference.html#tasks">Task APIs</a>. For streaming ingestion, tasks are generally submitted for you by a |
| supervisor.</p> |
| <h2><a class="anchor" aria-hidden="true" id="task-api"></a><a href="#task-api" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Task API</h2> |
| <p>Task APIs are available in two main places:</p> |
| <ul> |
| <li>The <a href="/docs/0.23.0/design/overlord.html">Overlord</a> process offers HTTP APIs to submit tasks, cancel tasks, check their status, |
| review logs and reports, and more. Refer to the <a href="/docs/0.23.0/operations/api-reference.html#tasks">Tasks API reference page</a> for a |
| full list.</li> |
| <li>Druid SQL includes a <a href="/docs/0.23.0/querying/sql-metadata-tables.html#tasks-table"><code>sys.tasks</code></a> table that provides information about currently |
| running tasks. This table is read-only, and has a limited (but useful!) subset of the full information available through |
| the Overlord APIs.</li> |
| </ul> |
| <p><a name="reports"></a></p> |
| <h2><a class="anchor" aria-hidden="true" id="task-reports"></a><a href="#task-reports" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Task reports</h2> |
| <p>A report containing information about the number of rows ingested, and any parse exceptions that occurred is available for both completed tasks and running tasks.</p> |
| <p>The reporting feature is supported by the <a href="/docs/0.23.0/ingestion/native-batch-simple-task.html">simple native batch task</a>, the Hadoop batch task, and Kafka and Kinesis ingestion tasks.</p> |
| <h3><a class="anchor" aria-hidden="true" id="completion-report"></a><a href="#completion-report" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Completion report</h3> |
| <p>After a task completes, a completion report can be retrieved at:</p> |
| <pre><code class="hljs">http:<span class="hljs-regexp">//</span><OVERLORD-HOST>:<OVERLORD-PORT><span class="hljs-regexp">/druid/i</span>ndexer<span class="hljs-regexp">/v1/</span>task<span class="hljs-regexp">/<task-id>/</span>reports |
| </code></pre> |
| <p>An example output is shown below:</p> |
| <pre><code class="hljs css language-json">{ |
| <span class="hljs-attr">"ingestionStatsAndErrors"</span>: { |
| <span class="hljs-attr">"taskId"</span>: <span class="hljs-string">"compact_twitter_2018-09-24T18:24:23.920Z"</span>, |
| <span class="hljs-attr">"payload"</span>: { |
| <span class="hljs-attr">"ingestionState"</span>: <span class="hljs-string">"COMPLETED"</span>, |
| <span class="hljs-attr">"unparseableEvents"</span>: {}, |
| <span class="hljs-attr">"rowStats"</span>: { |
| <span class="hljs-attr">"determinePartitions"</span>: { |
| <span class="hljs-attr">"processed"</span>: <span class="hljs-number">0</span>, |
| <span class="hljs-attr">"processedWithError"</span>: <span class="hljs-number">0</span>, |
| <span class="hljs-attr">"thrownAway"</span>: <span class="hljs-number">0</span>, |
| <span class="hljs-attr">"unparseable"</span>: <span class="hljs-number">0</span> |
| }, |
| <span class="hljs-attr">"buildSegments"</span>: { |
| <span class="hljs-attr">"processed"</span>: <span class="hljs-number">5390324</span>, |
| <span class="hljs-attr">"processedWithError"</span>: <span class="hljs-number">0</span>, |
| <span class="hljs-attr">"thrownAway"</span>: <span class="hljs-number">0</span>, |
| <span class="hljs-attr">"unparseable"</span>: <span class="hljs-number">0</span> |
| } |
| }, |
| <span class="hljs-attr">"segmentAvailabilityConfirmed"</span>: <span class="hljs-literal">false</span>, |
| <span class="hljs-attr">"segmentAvailabilityWaitTimeMs"</span>: <span class="hljs-number">0</span>, |
| <span class="hljs-attr">"errorMsg"</span>: <span class="hljs-literal">null</span> |
| }, |
| <span class="hljs-attr">"type"</span>: <span class="hljs-string">"ingestionStatsAndErrors"</span> |
| } |
| } |
| </code></pre> |
| <h4><a class="anchor" aria-hidden="true" id="segment-availability-fields"></a><a href="#segment-availability-fields" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Segment Availability Fields</h4> |
| <p>For some task types, the indexing task can wait for the newly ingested segments to become available for queries after ingestion completes. The below fields inform the end user regarding the duration and result of the availability wait. For batch ingestion task types, refer to <code>tuningConfig</code> docs to see if the task supports an availability waiting period.</p> |
| <table> |
| <thead> |
| <tr><th>Field</th><th>Description</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>segmentAvailabilityConfirmed</code></td><td>Whether all segments generated by this ingestion task had been confirmed as available for queries in the cluster before the task completed.</td></tr> |
| <tr><td><code>segmentAvailabilityWaitTimeMs</code></td><td>Milliseconds waited by the ingestion task for the newly ingested segments to be available for query after completing ingestion was completed.</td></tr> |
| </tbody> |
| </table> |
| <h3><a class="anchor" aria-hidden="true" id="live-report"></a><a href="#live-report" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Live report</h3> |
| <p>When a task is running, a live report containing ingestion state, unparseable events and moving average for number of events processed for 1 min, 5 min, 15 min time window can be retrieved at:</p> |
| <pre><code class="hljs">http:<span class="hljs-regexp">//</span><OVERLORD-HOST>:<OVERLORD-PORT><span class="hljs-regexp">/druid/i</span>ndexer<span class="hljs-regexp">/v1/</span>task<span class="hljs-regexp">/<task-id>/</span>reports |
| </code></pre> |
| <p>and</p> |
| <pre><code class="hljs">http:<span class="hljs-regexp">//</span><middlemanager-host>:<worker-port><span class="hljs-regexp">/druid/</span>worker<span class="hljs-regexp">/v1/</span>chat<span class="hljs-regexp">/<task-id>/</span>liveReports |
| </code></pre> |
| <p>An example output is shown below:</p> |
| <pre><code class="hljs css language-json">{ |
| <span class="hljs-attr">"ingestionStatsAndErrors"</span>: { |
| <span class="hljs-attr">"taskId"</span>: <span class="hljs-string">"compact_twitter_2018-09-24T18:24:23.920Z"</span>, |
| <span class="hljs-attr">"payload"</span>: { |
| <span class="hljs-attr">"ingestionState"</span>: <span class="hljs-string">"RUNNING"</span>, |
| <span class="hljs-attr">"unparseableEvents"</span>: {}, |
| <span class="hljs-attr">"rowStats"</span>: { |
| <span class="hljs-attr">"movingAverages"</span>: { |
| <span class="hljs-attr">"buildSegments"</span>: { |
| <span class="hljs-attr">"5m"</span>: { |
| <span class="hljs-attr">"processed"</span>: <span class="hljs-number">3.392158326408501</span>, |
| <span class="hljs-attr">"unparseable"</span>: <span class="hljs-number">0</span>, |
| <span class="hljs-attr">"thrownAway"</span>: <span class="hljs-number">0</span>, |
| <span class="hljs-attr">"processedWithError"</span>: <span class="hljs-number">0</span> |
| }, |
| <span class="hljs-attr">"15m"</span>: { |
| <span class="hljs-attr">"processed"</span>: <span class="hljs-number">1.736165476881023</span>, |
| <span class="hljs-attr">"unparseable"</span>: <span class="hljs-number">0</span>, |
| <span class="hljs-attr">"thrownAway"</span>: <span class="hljs-number">0</span>, |
| <span class="hljs-attr">"processedWithError"</span>: <span class="hljs-number">0</span> |
| }, |
| <span class="hljs-attr">"1m"</span>: { |
| <span class="hljs-attr">"processed"</span>: <span class="hljs-number">4.206417693750045</span>, |
| <span class="hljs-attr">"unparseable"</span>: <span class="hljs-number">0</span>, |
| <span class="hljs-attr">"thrownAway"</span>: <span class="hljs-number">0</span>, |
| <span class="hljs-attr">"processedWithError"</span>: <span class="hljs-number">0</span> |
| } |
| } |
| }, |
| <span class="hljs-attr">"totals"</span>: { |
| <span class="hljs-attr">"buildSegments"</span>: { |
| <span class="hljs-attr">"processed"</span>: <span class="hljs-number">1994</span>, |
| <span class="hljs-attr">"processedWithError"</span>: <span class="hljs-number">0</span>, |
| <span class="hljs-attr">"thrownAway"</span>: <span class="hljs-number">0</span>, |
| <span class="hljs-attr">"unparseable"</span>: <span class="hljs-number">0</span> |
| } |
| } |
| }, |
| <span class="hljs-attr">"errorMsg"</span>: <span class="hljs-literal">null</span> |
| }, |
| <span class="hljs-attr">"type"</span>: <span class="hljs-string">"ingestionStatsAndErrors"</span> |
| } |
| } |
| </code></pre> |
| <p>A description of the fields:</p> |
| <p>The <code>ingestionStatsAndErrors</code> report provides information about row counts and errors.</p> |
| <p>The <code>ingestionState</code> shows what step of ingestion the task reached. Possible states include:</p> |
| <ul> |
| <li><code>NOT_STARTED</code>: The task has not begun reading any rows</li> |
| <li><code>DETERMINE_PARTITIONS</code>: The task is processing rows to determine partitioning</li> |
| <li><code>BUILD_SEGMENTS</code>: The task is processing rows to construct segments</li> |
| <li><code>COMPLETED</code>: The task has finished its work.</li> |
| </ul> |
| <p>Only batch tasks have the DETERMINE_PARTITIONS phase. Realtime tasks such as those created by the Kafka Indexing Service do not have a DETERMINE_PARTITIONS phase.</p> |
| <p><code>unparseableEvents</code> contains lists of exception messages that were caused by unparseable inputs. This can help with identifying problematic input rows. There will be one list each for the DETERMINE_PARTITIONS and BUILD_SEGMENTS phases. Note that the Hadoop batch task does not support saving of unparseable events.</p> |
| <p>the <code>rowStats</code> map contains information about row counts. There is one entry for each ingestion phase. The definitions of the different row counts are shown below:</p> |
| <ul> |
| <li><code>processed</code>: Number of rows successfully ingested without parsing errors</li> |
| <li><code>processedWithError</code>: Number of rows that were ingested, but contained a parsing error within one or more columns. This typically occurs where input rows have a parseable structure but invalid types for columns, such as passing in a non-numeric String value for a numeric column.</li> |
| <li><code>thrownAway</code>: Number of rows skipped. This includes rows with timestamps that were outside of the ingestion task's defined time interval and rows that were filtered out with a <a href="/docs/0.23.0/ingestion/ingestion-spec.html#transformspec"><code>transformSpec</code></a>, but doesn't include the rows skipped by explicit user configurations. For example, the rows skipped by <code>skipHeaderRows</code> or <code>hasHeaderRow</code> in the CSV format are not counted.</li> |
| <li><code>unparseable</code>: Number of rows that could not be parsed at all and were discarded. This tracks input rows without a parseable structure, such as passing in non-JSON data when using a JSON parser.</li> |
| </ul> |
| <p>The <code>errorMsg</code> field shows a message describing the error that caused a task to fail. It will be null if the task was successful.</p> |
| <h2><a class="anchor" aria-hidden="true" id="live-reports"></a><a href="#live-reports" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Live reports</h2> |
| <h3><a class="anchor" aria-hidden="true" id="row-stats"></a><a href="#row-stats" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Row stats</h3> |
| <p>The non-parallel <a href="/docs/0.23.0/ingestion/native-batch-simple-task.html">simple native batch task</a>, the Hadoop batch task, and Kafka and Kinesis ingestion tasks support retrieval of row stats while the task is running.</p> |
| <p>The live report can be accessed with a GET to the following URL on a Peon running a task:</p> |
| <pre><code class="hljs">http:<span class="hljs-regexp">//</span><middlemanager-host>:<worker-port><span class="hljs-regexp">/druid/</span>worker<span class="hljs-regexp">/v1/</span>chat<span class="hljs-regexp">/<task-id>/</span>rowStats |
| </code></pre> |
| <p>An example report is shown below. The <code>movingAverages</code> section contains 1 minute, 5 minute, and 15 minute moving averages of increases to the four row counters, which have the same definitions as those in the completion report. The <code>totals</code> section shows the current totals.</p> |
| <pre><code class="hljs">{ |
| <span class="hljs-attr">"movingAverages"</span>: { |
| <span class="hljs-attr">"buildSegments"</span>: { |
| <span class="hljs-attr">"5m"</span>: { |
| <span class="hljs-attr">"processed"</span>: <span class="hljs-number">3.392158326408501</span>, |
| <span class="hljs-attr">"unparseable"</span>: <span class="hljs-number">0</span>, |
| <span class="hljs-attr">"thrownAway"</span>: <span class="hljs-number">0</span>, |
| <span class="hljs-attr">"processedWithError"</span>: <span class="hljs-number">0</span> |
| }, |
| <span class="hljs-attr">"15m"</span>: { |
| <span class="hljs-attr">"processed"</span>: <span class="hljs-number">1.736165476881023</span>, |
| <span class="hljs-attr">"unparseable"</span>: <span class="hljs-number">0</span>, |
| <span class="hljs-attr">"thrownAway"</span>: <span class="hljs-number">0</span>, |
| <span class="hljs-attr">"processedWithError"</span>: <span class="hljs-number">0</span> |
| }, |
| <span class="hljs-attr">"1m"</span>: { |
| <span class="hljs-attr">"processed"</span>: <span class="hljs-number">4.206417693750045</span>, |
| <span class="hljs-attr">"unparseable"</span>: <span class="hljs-number">0</span>, |
| <span class="hljs-attr">"thrownAway"</span>: <span class="hljs-number">0</span>, |
| <span class="hljs-attr">"processedWithError"</span>: <span class="hljs-number">0</span> |
| } |
| } |
| }, |
| <span class="hljs-attr">"totals"</span>: { |
| <span class="hljs-attr">"buildSegments"</span>: { |
| <span class="hljs-attr">"processed"</span>: <span class="hljs-number">1994</span>, |
| <span class="hljs-attr">"processedWithError"</span>: <span class="hljs-number">0</span>, |
| <span class="hljs-attr">"thrownAway"</span>: <span class="hljs-number">0</span>, |
| <span class="hljs-attr">"unparseable"</span>: <span class="hljs-number">0</span> |
| } |
| } |
| } |
| </code></pre> |
| <p>For the Kafka Indexing Service, a GET to the following Overlord API will retrieve live row stat reports from each task being managed by the supervisor and provide a combined report.</p> |
| <pre><code class="hljs">http:<span class="hljs-regexp">//</span><OVERLORD-HOST>:<OVERLORD-PORT><span class="hljs-regexp">/druid/i</span>ndexer<span class="hljs-regexp">/v1/</span>supervisor<span class="hljs-regexp">/<supervisor-id>/</span>stats |
| </code></pre> |
| <h3><a class="anchor" aria-hidden="true" id="unparseable-events"></a><a href="#unparseable-events" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Unparseable events</h3> |
| <p>Lists of recently-encountered unparseable events can be retrieved from a running task with a GET to the following Peon API:</p> |
| <pre><code class="hljs">http:<span class="hljs-regexp">//</span><middlemanager-host>:<worker-port><span class="hljs-regexp">/druid/</span>worker<span class="hljs-regexp">/v1/</span>chat<span class="hljs-regexp">/<task-id>/u</span>nparseableEvents |
| </code></pre> |
| <p>Note that this functionality is not supported by all task types. Currently, it is only supported by the |
| non-parallel <a href="/docs/0.23.0/ingestion/native-batch.html">native batch task</a> (type <code>index</code>) and the tasks created by the Kafka |
| and Kinesis indexing services.</p> |
| <p><a name="locks"></a></p> |
| <h2><a class="anchor" aria-hidden="true" id="task-lock-system"></a><a href="#task-lock-system" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Task lock system</h2> |
| <p>This section explains the task locking system in Druid. Druid's locking system |
| and versioning system are tightly coupled with each other to guarantee the correctness of ingested data.</p> |
| <h3><a class="anchor" aria-hidden="true" id="overshadowing-between-segments"></a><a href="#overshadowing-between-segments" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>"Overshadowing" between segments</h3> |
| <p>You can run a task to overwrite existing data. The segments created by an overwriting task <em>overshadows</em> existing segments. |
| Note that the overshadow relation holds only for the same time chunk and the same data source. |
| These overshadowed segments are not considered in query processing to filter out stale data.</p> |
| <p>Each segment has a <em>major</em> version and a <em>minor</em> version. The major version is |
| represented as a timestamp in the format of <a href="https://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat"><code>"yyyy-MM-dd'T'hh:mm:ss"</code></a> |
| while the minor version is an integer number. These major and minor versions |
| are used to determine the overshadow relation between segments as seen below.</p> |
| <p>A segment <code>s1</code> overshadows another <code>s2</code> if</p> |
| <ul> |
| <li><code>s1</code> has a higher major version than <code>s2</code>, or</li> |
| <li><code>s1</code> has the same major version and a higher minor version than <code>s2</code>.</li> |
| </ul> |
| <p>Here are some examples.</p> |
| <ul> |
| <li>A segment of the major version of <code>2019-01-01T00:00:00.000Z</code> and the minor version of <code>0</code> overshadows |
| another of the major version of <code>2018-01-01T00:00:00.000Z</code> and the minor version of <code>1</code>.</li> |
| <li>A segment of the major version of <code>2019-01-01T00:00:00.000Z</code> and the minor version of <code>1</code> overshadows |
| another of the major version of <code>2019-01-01T00:00:00.000Z</code> and the minor version of <code>0</code>.</li> |
| </ul> |
| <h3><a class="anchor" aria-hidden="true" id="locking"></a><a href="#locking" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Locking</h3> |
| <p>If you are running two or more <a href="/docs/0.23.0/ingestion/tasks.html">Druid tasks</a> which generate segments for the same data source and the same time chunk, |
| the generated segments could potentially overshadow each other, which could lead to incorrect query results.</p> |
| <p>To avoid this problem, tasks will attempt to get locks prior to creating any segment in Druid. |
| There are two types of locks, i.e., <em>time chunk lock</em> and <em>segment lock</em>.</p> |
| <p>When the time chunk lock is used, a task locks the entire time chunk of a data source where generated segments will be written. |
| For example, suppose we have a task ingesting data into the time chunk of <code>2019-01-01T00:00:00.000Z/2019-01-02T00:00:00.000Z</code> of the <code>wikipedia</code> data source. |
| With the time chunk locking, this task will lock the entire time chunk of <code>2019-01-01T00:00:00.000Z/2019-01-02T00:00:00.000Z</code> of the <code>wikipedia</code> data source |
| before it creates any segments. As long as it holds the lock, any other tasks will be unable to create segments for the same time chunk of the same data source. |
| The segments created with the time chunk locking have a <em>higher</em> major version than existing segments. Their minor version is always <code>0</code>.</p> |
| <p>When the segment lock is used, a task locks individual segments instead of the entire time chunk. |
| As a result, two or more tasks can create segments for the same time chunk of the same data source simultaneously |
| if they are reading different segments. |
| For example, a Kafka indexing task and a compaction task can always write segments into the same time chunk of the same data source simultaneously. |
| The reason for this is because a Kafka indexing task always appends new segments, while a compaction task always overwrites existing segments. |
| The segments created with the segment locking have the <em>same</em> major version and a <em>higher</em> minor version.</p> |
| <blockquote> |
| <p>The segment locking is still experimental. It could have unknown bugs which potentially lead to incorrect query results.</p> |
| </blockquote> |
| <p>To enable segment locking, you may need to set <code>forceTimeChunkLock</code> to <code>false</code> in the <a href="#context">task context</a>. |
| Once <code>forceTimeChunkLock</code> is unset, the task will choose a proper lock type to use automatically. |
| Please note that segment lock is not always available. The most common use case where time chunk lock is enforced is |
| when an overwriting task changes the segment granularity. |
| Also, the segment locking is supported by only native indexing tasks and Kafka/Kinesis indexing tasks. |
| Hadoop indexing tasks don't support it.</p> |
| <p><code>forceTimeChunkLock</code> in the task context is only applied to individual tasks. |
| If you want to unset it for all tasks, you would want to set <code>druid.indexer.tasklock.forceTimeChunkLock</code> to false in the <a href="/docs/0.23.0/configuration/index.html#overlord-operations">overlord configuration</a>.</p> |
| <p>Lock requests can conflict with each other if two or more tasks try to get locks for the overlapped time chunks of the same data source. |
| Note that the lock conflict can happen between different locks types.</p> |
| <p>The behavior on lock conflicts depends on the <a href="#lock-priority">task priority</a>. |
| If all tasks of conflicting lock requests have the same priority, then the task who requested first will get the lock. |
| Other tasks will wait for the task to release the lock.</p> |
| <p>If a task of a lower priority asks a lock later than another of a higher priority, |
| this task will also wait for the task of a higher priority to release the lock. |
| If a task of a higher priority asks a lock later than another of a lower priority, |
| then this task will <em>preempt</em> the other task of a lower priority. The lock |
| of the lower-prioritized task will be revoked and the higher-prioritized task will acquire a new lock.</p> |
| <p>This lock preemption can happen at any time while a task is running except |
| when it is <em>publishing segments</em> in a critical section. Its locks become preemptible again once publishing segments is finished.</p> |
| <p>Note that locks are shared by the tasks of the same groupId. |
| For example, Kafka indexing tasks of the same supervisor have the same groupId and share all locks with each other.</p> |
| <p><a name="priority"></a></p> |
| <h3><a class="anchor" aria-hidden="true" id="lock-priority"></a><a href="#lock-priority" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Lock priority</h3> |
| <p>Each task type has a different default lock priority. The below table shows the default priorities of different task types. Higher the number, higher the priority.</p> |
| <table> |
| <thead> |
| <tr><th>task type</th><th>default priority</th></tr> |
| </thead> |
| <tbody> |
| <tr><td>Realtime index task</td><td>75</td></tr> |
| <tr><td>Batch index task</td><td>50</td></tr> |
| <tr><td>Merge/Append/Compaction task</td><td>25</td></tr> |
| <tr><td>Other tasks</td><td>0</td></tr> |
| </tbody> |
| </table> |
| <p>You can override the task priority by setting your priority in the task context as below.</p> |
| <pre><code class="hljs css language-json">"context" : { |
| "priority" : 100 |
| } |
| </code></pre> |
| <p><a name="context"></a></p> |
| <h2><a class="anchor" aria-hidden="true" id="context-parameters"></a><a href="#context-parameters" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Context parameters</h2> |
| <p>The task context is used for various individual task configuration. |
| Specify task context configurations in the <code>context</code> field of the ingestion spec. |
| The following parameters apply to all task types.</p> |
| <table> |
| <thead> |
| <tr><th>property</th><th>default</th><th>description</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>taskLockTimeout</code></td><td>300000</td><td>Task lock timeout in milliseconds. For more details, see <a href="#locking">Locking</a>.<br/><br/>When a task acquires a lock, it sends a request via HTTP and awaits until it receives a response containing the lock acquisition result. As a result, an HTTP timeout error can occur if <code>taskLockTimeout</code> is greater than <code>druid.server.http.maxIdleTime</code> of Overlords.</td></tr> |
| <tr><td><code>forceTimeChunkLock</code></td><td>true</td><td><em>Setting this to false is still experimental</em><br/> Force to always use time chunk lock. If not set, each task automatically chooses a lock type to use. If set, this parameter overwrites <code>druid.indexer.tasklock.forceTimeChunkLock</code> <a href="/docs/0.23.0/configuration/index.html#overlord-operations">configuration for the overlord</a>. See <a href="#locking">Locking</a> for more details.</td></tr> |
| <tr><td><code>priority</code></td><td>Different based on task types. See <a href="#priority">Priority</a>.</td><td>Task priority</td></tr> |
| <tr><td><code>useLineageBasedSegmentAllocation</code></td><td>false in 0.21 or earlier, true in 0.22 or later</td><td>Enable the new lineage-based segment allocation protocol for the native Parallel task with dynamic partitioning. This option should be off during the replacing rolling upgrade from one of the Druid versions between 0.19 and 0.21 to Druid 0.22 or higher. Once the upgrade is done, it must be set to true to ensure data correctness.</td></tr> |
| <tr><td><code>storeEmptyColumns</code></td><td>true</td><td>Boolean value for whether or not to store empty columns during ingestion. When set to true, Druid stores every column specified in the <a href="/docs/0.23.0/ingestion/ingestion-spec.html#dimensionsspec"><code>dimensionsSpec</code></a>. If you use schemaless ingestion and don't specify any dimensions to ingest, you must also set <a href="/docs/0.23.0/ingestion/ingestion-spec.html#dimensionsspec"><code>includeAllDimensions</code></a> for Druid to store empty columns.<br/><br/>If you set <code>storeEmptyColumns</code> to false, Druid SQL queries referencing empty columns will fail. If you intend to leave <code>storeEmptyColumns</code> disabled, you should either ingest dummy data for empty columns or else not query on empty columns.<br/><br/>When set in the task context, <code>storeEmptyColumns</code> overrides the system property <a href="/docs/0.23.0/configuration/index.html#additional-peon-configuration"><code>druid.indexer.task.storeEmptyColumns</code></a>.</td></tr> |
| </tbody> |
| </table> |
| <h2><a class="anchor" aria-hidden="true" id="task-logs"></a><a href="#task-logs" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Task logs</h2> |
| <p>Logs are created by ingestion tasks as they run. You can configure Druid to push these into a repository for long-term storage after they complete.</p> |
| <p>Once the task has been submitted to the Overlord it remains <code>WAITING</code> for locks to be acquired. Worker slot allocation is then <code>PENDING</code> until the task can actually start executing.</p> |
| <p>The task then starts creating logs in a local directory of the middle manager (or indexer) in a <code>log</code> directory for the specific <code>taskId</code> at <a href="/docs/0.23.0/configuration/index.html#additional-peon-configuration"><code>druid.indexer.task.baseTaskDir</code></a>.</p> |
| <p>When the task completes - whether it succeeds or fails - the middle manager (or indexer) will push the task log file into the location specified in <a href="/docs/0.23.0/configuration/index.html#task-logging"><code>druid.indexer.logs</code></a>.</p> |
| <p>Task logs on the Druid web console are retrieved via an <a href="/docs/0.23.0/operations/api-reference.html#overlord">API</a> on the Overlord. It automatically detects where the log file is, either in the middleManager / indexer or in long-term storage, and passes it back.</p> |
| <p>If you don't see the log file in long-term storage, it means either:</p> |
| <ol> |
| <li>the middleManager / indexer failed to push the log file to deep storage or</li> |
| <li>the task did not complete.</li> |
| </ol> |
| <p>You can check the middleManager / indexer logs locally to see if there was a push failure. If there was not, check the Overlord's own process logs to see why the task failed before it started.</p> |
| <blockquote> |
| <p>If you are running the indexing service in remote mode, the task logs must be stored in S3, Azure Blob Store, Google Cloud Storage or HDFS.</p> |
| </blockquote> |
| <p>You can configure retention periods for logs in milliseconds by setting <code>druid.indexer.logs.kill</code> properties in <a href="/docs/0.23.0/configuration/index.html#task-logging">configuration</a>. The Overlord will then automatically manage task logs in log directories along with entries in task-related metadata storage tables.</p> |
| <blockquote> |
| <p>Automatic log file deletion typically works based on the log file's 'modified' timestamp in the back-end store. Large clock skews between Druid processes and the long-term store might result in unintended behavior.</p> |
| </blockquote> |
| <h2><a class="anchor" aria-hidden="true" id="all-task-types"></a><a href="#all-task-types" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>All task types</h2> |
| <h3><a class="anchor" aria-hidden="true" id="index"></a><a href="#index" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a><code>index</code></h3> |
| <p>See <a href="/docs/0.23.0/ingestion/native-batch-simple-task.html">Native batch ingestion (simple task)</a>.</p> |
| <h3><a class="anchor" aria-hidden="true" id="index_parallel"></a><a href="#index_parallel" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a><code>index_parallel</code></h3> |
| <p>See <a href="/docs/0.23.0/ingestion/native-batch.html">Native batch ingestion (parallel task)</a>.</p> |
| <h3><a class="anchor" aria-hidden="true" id="index_sub"></a><a href="#index_sub" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a><code>index_sub</code></h3> |
| <p>Submitted automatically, on your behalf, by an <a href="#index_parallel"><code>index_parallel</code></a> task.</p> |
| <h3><a class="anchor" aria-hidden="true" id="index_hadoop"></a><a href="#index_hadoop" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a><code>index_hadoop</code></h3> |
| <p>See <a href="/docs/0.23.0/ingestion/hadoop.html">Hadoop-based ingestion</a>.</p> |
| <h3><a class="anchor" aria-hidden="true" id="index_kafka"></a><a href="#index_kafka" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a><code>index_kafka</code></h3> |
| <p>Submitted automatically, on your behalf, by a |
| <a href="/docs/0.23.0/development/extensions-core/kafka-ingestion.html">Kafka-based ingestion supervisor</a>.</p> |
| <h3><a class="anchor" aria-hidden="true" id="index_kinesis"></a><a href="#index_kinesis" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a><code>index_kinesis</code></h3> |
| <p>Submitted automatically, on your behalf, by a |
| <a href="/docs/0.23.0/development/extensions-core/kinesis-ingestion.html">Kinesis-based ingestion supervisor</a>.</p> |
| <h3><a class="anchor" aria-hidden="true" id="index_realtime"></a><a href="#index_realtime" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a><code>index_realtime</code></h3> |
| <p>Submitted automatically, on your behalf, by <a href="/docs/0.23.0/ingestion/tranquility.html">Tranquility</a>.</p> |
| <h3><a class="anchor" aria-hidden="true" id="compact"></a><a href="#compact" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a><code>compact</code></h3> |
| <p>Compaction tasks merge all segments of the given interval. See the documentation on |
| <a href="/docs/0.23.0/ingestion/compaction.html">compaction</a> for details.</p> |
| <h3><a class="anchor" aria-hidden="true" id="kill"></a><a href="#kill" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a><code>kill</code></h3> |
| <p>Kill tasks delete all metadata about certain segments and removes them from deep storage. |
| See the documentation on <a href="/docs/0.23.0/ingestion/data-management.html#delete">deleting data</a> for details.</p> |
| </span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/0.23.0/ingestion/hadoop.html"><span class="arrow-prev">← </span><span>Hadoop-based</span></a><a class="docs-next button" href="/docs/0.23.0/ingestion/faq.html"><span>Troubleshooting FAQ</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#task-api">Task API</a></li><li><a href="#task-reports">Task reports</a><ul class="toc-headings"><li><a href="#completion-report">Completion report</a></li><li><a href="#live-report">Live report</a></li></ul></li><li><a href="#live-reports">Live reports</a><ul class="toc-headings"><li><a href="#row-stats">Row stats</a></li><li><a href="#unparseable-events">Unparseable events</a></li></ul></li><li><a href="#task-lock-system">Task lock system</a><ul class="toc-headings"><li><a href="#overshadowing-between-segments">"Overshadowing" between segments</a></li><li><a href="#locking">Locking</a></li><li><a href="#lock-priority">Lock priority</a></li></ul></li><li><a href="#context-parameters">Context parameters</a></li><li><a href="#task-logs">Task logs</a></li><li><a href="#all-task-types">All task types</a><ul class="toc-headings"><li><a href="#index"><code>index</code></a></li><li><a href="#index_parallel"><code>index_parallel</code></a></li><li><a href="#index_sub"><code>index_sub</code></a></li><li><a href="#index_hadoop"><code>index_hadoop</code></a></li><li><a href="#index_kafka"><code>index_kafka</code></a></li><li><a href="#index_kinesis"><code>index_kinesis</code></a></li><li><a href="#index_realtime"><code>index_realtime</code></a></li><li><a href="#compact"><code>compact</code></a></li><li><a href="#kill"><code>kill</code></a></li></ul></li></ul></nav></div><footer class="nav-footer druid-footer" id="footer"><div class="container"><div class="text-center"><p><a href="/technology">Technology</a> · <a href="/use-cases">Use Cases</a> · <a href="/druid-powered">Powered by Druid</a> · <a href="/docs/0.23.0/latest">Docs</a> · <a href="/community/">Community</a> · <a href="/downloads.html">Download</a> · <a href="/faq">FAQ</a></p></div><div class="text-center"><a title="Join the user group" href="https://groups.google.com/forum/#!forum/druid-user" target="_blank"><span class="fa fa-comments"></span></a> · <a title="Follow Druid" href="https://twitter.com/druidio" target="_blank"><span class="fab fa-twitter"></span></a> · <a title="Download via Apache" href="https://www.apache.org/dyn/closer.cgi?path=/incubator/druid/{{ site.druid_versions[0].versions[0].version }}/apache-druid-{{ site.druid_versions[0].versions[0].version }}-bin.tar.gz" target="_blank"><span class="fas fa-feather"></span></a> · <a title="GitHub" href="https://github.com/apache/druid" target="_blank"><span class="fab fa-github"></span></a></div><div class="text-center license">Copyright © 2019 <a href="https://www.apache.org/" target="_blank">Apache Software Foundation</a>.<br/>Except where otherwise noted, licensed under <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">CC BY-SA 4.0</a>.<br/>Apache Druid, Druid, and the Druid logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</div></div></footer></div><script type="text/javascript" src="https://cdn.jsdelivr.net/docsearch.js/1/docsearch.min.js"></script><script> |
| document.addEventListener('keyup', function(e) { |
| if (e.target !== document.body) { |
| return; |
| } |
| // keyCode for '/' (slash) |
| if (e.keyCode === 191) { |
| const search = document.getElementById('search_input_react'); |
| search && search.focus(); |
| } |
| }); |
| </script><script> |
| var search = docsearch({ |
| |
| apiKey: '2de99082a9f38e49dfaa059bbe4c901d', |
| indexName: 'apache_druid', |
| inputSelector: '#search_input_react', |
| algoliaOptions: {"facetFilters":["language:en","version:0.23.0"]} |
| }); |
| </script></body></html> |
