| <!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Configuration reference · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/latest/configuration/index.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="<!--"/><meta name="docsearch:language" content="en"/><meta name="docsearch:version" content="0.20.0" /><meta property="og:title" content="Configuration reference · Apache Druid"/><meta property="og:type" content="website"/><meta property="og:url" content="https://druid.apache.org/index.html"/><meta property="og:description" content="<!--"/><meta property="og:image" content="https://druid.apache.org/img/druid_nav.png"/><meta name="twitter:card" content="summary"/><meta name="twitter:image" content="https://druid.apache.org/img/druid_nav.png"/><link rel="shortcut icon" href="/img/favicon.png"/><link rel="stylesheet" href="https://cdn.jsdelivr.net/docsearch.js/1/docsearch.min.css"/><link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/default.min.css"/><script async="" src="https://www.googletagmanager.com/gtag/js?id=UA-131010415-1"></script><script> |
| window.dataLayer = window.dataLayer || []; |
| function gtag(){dataLayer.push(arguments); } |
| gtag('js', new Date()); |
| gtag('config', 'UA-131010415-1'); |
| </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/druid_nav.png" alt="Apache Druid"/></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class=""><a href="/technology" target="_self">Technology</a></li><li class=""><a href="/use-cases" target="_self">Use Cases</a></li><li class=""><a href="/druid-powered" target="_self">Powered By</a></li><li class="siteNavGroupActive"><a href="/docs/latest/design/index.html" target="_self">Docs</a></li><li class=""><a href="/community/" target="_self">Community</a></li><li class=""><a href="https://www.apache.org" target="_self">Apache</a></li><li class=""><a href="/downloads.html" target="_self">Download</a></li><li class="navSearchWrapper reactNavSearchWrapper"><input type="text" id="search_input_react" placeholder="Search" title="Search"/></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Configuration</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle collapsible">Getting started<span class="arrow"><svg width="24" height="24" viewBox="0 0 24 24"><path fill="#565656" d="M7.41 15.41L12 10.83l4.59 4.58L18 14l-6-6-6 6z"></path><path d="M0 0h24v24H0z" fill="none"></path></svg></span></h3><ul class="hide"><li class="navListItem"><a class="navItem" href="/docs/latest/design/index.html">Introduction to Apache Druid</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/tutorials/index.html">Quickstart</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/tutorials/docker.html">Docker</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/operations/single-server.html">Single server deployment</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/tutorials/cluster.html">Clustered deployment</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle collapsible">Tutorials<span class="arrow"><svg width="24" height="24" viewBox="0 0 24 24"><path fill="#565656" d="M7.41 15.41L12 10.83l4.59 4.58L18 14l-6-6-6 6z"></path><path d="M0 0h24v24H0z" fill="none"></path></svg></span></h3><ul class="hide"><li class="navListItem"><a class="navItem" href="/docs/latest/tutorials/tutorial-batch.html">Loading files natively</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/tutorials/tutorial-kafka.html">Load from Apache Kafka</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/tutorials/tutorial-batch-hadoop.html">Load from Apache Hadoop</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/tutorials/tutorial-query.html">Querying data</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/tutorials/tutorial-rollup.html">Roll-up</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/tutorials/tutorial-retention.html">Configuring data retention</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/tutorials/tutorial-update-data.html">Updating existing data</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/tutorials/tutorial-compaction.html">Compacting segments</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/tutorials/tutorial-delete-data.html">Deleting data</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/tutorials/tutorial-ingestion-spec.html">Writing an ingestion spec</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/tutorials/tutorial-transform-spec.html">Transforming input data</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/tutorials/tutorial-kerberos-hadoop.html">Kerberized HDFS deep storage</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle collapsible">Design<span class="arrow"><svg width="24" height="24" viewBox="0 0 24 24"><path fill="#565656" d="M7.41 15.41L12 10.83l4.59 4.58L18 14l-6-6-6 6z"></path><path d="M0 0h24v24H0z" fill="none"></path></svg></span></h3><ul class="hide"><li class="navListItem"><a class="navItem" href="/docs/latest/design/architecture.html">Design</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/design/segments.html">Segments</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/design/processes.html">Processes and servers</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/dependencies/deep-storage.html">Deep storage</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/dependencies/metadata-storage.html">Metadata storage</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/dependencies/zookeeper.html">ZooKeeper</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle collapsible">Ingestion<span class="arrow"><svg width="24" height="24" viewBox="0 0 24 24"><path fill="#565656" d="M7.41 15.41L12 10.83l4.59 4.58L18 14l-6-6-6 6z"></path><path d="M0 0h24v24H0z" fill="none"></path></svg></span></h3><ul class="hide"><li class="navListItem"><a class="navItem" href="/docs/latest/ingestion/index.html">Ingestion</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/ingestion/data-formats.html">Data formats</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/ingestion/schema-design.html">Schema design tips</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/ingestion/data-management.html">Data management</a></li><div class="navGroup subNavGroup"><h4 class="navGroupSubcategoryTitle">Stream ingestion</h4><ul><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-core/kafka-ingestion.html">Apache Kafka</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-core/kinesis-ingestion.html">Amazon Kinesis</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/ingestion/tranquility.html">Tranquility</a></li></ul></div><div class="navGroup subNavGroup"><h4 class="navGroupSubcategoryTitle">Batch ingestion</h4><ul><li class="navListItem"><a class="navItem" href="/docs/latest/ingestion/native-batch.html">Native batch</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/ingestion/hadoop.html">Hadoop-based</a></li></ul></div><li class="navListItem"><a class="navItem" href="/docs/latest/ingestion/tasks.html">Task reference</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/ingestion/faq.html">Troubleshooting FAQ</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle collapsible">Querying<span class="arrow"><svg width="24" height="24" viewBox="0 0 24 24"><path fill="#565656" d="M7.41 15.41L12 10.83l4.59 4.58L18 14l-6-6-6 6z"></path><path d="M0 0h24v24H0z" fill="none"></path></svg></span></h3><ul class="hide"><li class="navListItem"><a class="navItem" href="/docs/latest/querying/sql.html">Druid SQL</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/querying/querying.html">Native queries</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/querying/query-execution.html">Query execution</a></li><div class="navGroup subNavGroup"><h4 class="navGroupSubcategoryTitle">Concepts</h4><ul><li class="navListItem"><a class="navItem" href="/docs/latest/querying/datasource.html">Datasources</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/querying/joins.html">Joins</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/querying/lookups.html">Lookups</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/querying/multi-value-dimensions.html">Multi-value dimensions</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/querying/multitenancy.html">Multitenancy</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/querying/caching.html">Query caching</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/querying/query-context.html">Context parameters</a></li></ul></div><div class="navGroup subNavGroup"><h4 class="navGroupSubcategoryTitle">Native query types</h4><ul><li class="navListItem"><a class="navItem" href="/docs/latest/querying/timeseriesquery.html">Timeseries</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/querying/topnquery.html">TopN</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/querying/groupbyquery.html">GroupBy</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/querying/scan-query.html">Scan</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/querying/searchquery.html">Search</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/querying/timeboundaryquery.html">TimeBoundary</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/querying/segmentmetadataquery.html">SegmentMetadata</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/querying/datasourcemetadataquery.html">DatasourceMetadata</a></li></ul></div><div class="navGroup subNavGroup"><h4 class="navGroupSubcategoryTitle">Native query components</h4><ul><li class="navListItem"><a class="navItem" href="/docs/latest/querying/filters.html">Filters</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/querying/granularities.html">Granularities</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/querying/dimensionspecs.html">Dimensions</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/querying/aggregations.html">Aggregations</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/querying/post-aggregations.html">Post-aggregations</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/misc/math-expr.html">Expressions</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/querying/having.html">Having filters (groupBy)</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/querying/limitspec.html">Sorting and limiting (groupBy)</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/querying/topnmetricspec.html">Sorting (topN)</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/querying/sorting-orders.html">String comparators</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/querying/virtual-columns.html">Virtual columns</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/geo.html">Spatial filters</a></li></ul></div></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle collapsible">Configuration<span class="arrow"><svg width="24" height="24" viewBox="0 0 24 24"><path fill="#565656" d="M7.41 15.41L12 10.83l4.59 4.58L18 14l-6-6-6 6z"></path><path d="M0 0h24v24H0z" fill="none"></path></svg></span></h3><ul class="hide"><li class="navListItem navListItemActive"><a class="navItem" href="/docs/latest/configuration/index.html">Configuration reference</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions.html">Extensions</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/configuration/logging.html">Logging</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle collapsible">Operations<span class="arrow"><svg width="24" height="24" viewBox="0 0 24 24"><path fill="#565656" d="M7.41 15.41L12 10.83l4.59 4.58L18 14l-6-6-6 6z"></path><path d="M0 0h24v24H0z" fill="none"></path></svg></span></h3><ul class="hide"><li class="navListItem"><a class="navItem" href="/docs/latest/operations/druid-console.html">Web console</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/operations/getting-started.html">Getting started with Apache Druid</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/operations/basic-cluster-tuning.html">Basic cluster tuning</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/operations/api-reference.html">API reference</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/operations/high-availability.html">High availability</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/operations/rolling-updates.html">Rolling updates</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/operations/rule-configuration.html">Retaining or automatically dropping data</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/operations/metrics.html">Metrics</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/operations/alerts.html">Alerts</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/operations/other-hadoop.html">Working with different versions of Apache Hadoop</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/operations/http-compression.html">HTTP compression</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/operations/tls-support.html">TLS support</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/operations/password-provider.html">Password providers</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/operations/dump-segment.html">dump-segment tool</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/operations/reset-cluster.html">reset-cluster tool</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/operations/insert-segment-to-db.html">insert-segment-to-db tool</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/operations/pull-deps.html">pull-deps tool</a></li><div class="navGroup subNavGroup"><h4 class="navGroupSubcategoryTitle">Misc</h4><ul><li class="navListItem"><a class="navItem" href="/docs/latest/operations/management-uis.html">Legacy Management UIs</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/operations/deep-storage-migration.html">Deep storage migration</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/operations/export-metadata.html">Export Metadata Tool</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/operations/metadata-migration.html">Metadata Migration</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/operations/segment-optimization.html">Segment Size Optimization</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/operations/use_sbt_to_build_fat_jar.html">Content for build.sbt</a></li></ul></div></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle collapsible">Development<span class="arrow"><svg width="24" height="24" viewBox="0 0 24 24"><path fill="#565656" d="M7.41 15.41L12 10.83l4.59 4.58L18 14l-6-6-6 6z"></path><path d="M0 0h24v24H0z" fill="none"></path></svg></span></h3><ul class="hide"><li class="navListItem"><a class="navItem" href="/docs/latest/development/overview.html">Developing on Druid</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/modules.html">Creating extensions</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/javascript.html">JavaScript functionality</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/build.html">Build from source</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/versioning.html">Versioning</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/experimental.html">Experimental features</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle collapsible">Misc<span class="arrow"><svg width="24" height="24" viewBox="0 0 24 24"><path fill="#565656" d="M7.41 15.41L12 10.83l4.59 4.58L18 14l-6-6-6 6z"></path><path d="M0 0h24v24H0z" fill="none"></path></svg></span></h3><ul class="hide"><li class="navListItem"><a class="navItem" href="/docs/latest/misc/papers-and-talks.html">Papers</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle collapsible">Hidden<span class="arrow"><svg width="24" height="24" viewBox="0 0 24 24"><path fill="#565656" d="M7.41 15.41L12 10.83l4.59 4.58L18 14l-6-6-6 6z"></path><path d="M0 0h24v24H0z" fill="none"></path></svg></span></h3><ul class="hide"><li class="navListItem"><a class="navItem" href="/docs/latest/comparisons/druid-vs-elasticsearch.html">Apache Druid vs Elasticsearch</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/comparisons/druid-vs-key-value.html">Apache Druid vs. Key/Value Stores (HBase/Cassandra/OpenTSDB)</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/comparisons/druid-vs-kudu.html">Apache Druid vs Kudu</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/comparisons/druid-vs-redshift.html">Apache Druid vs Redshift</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/comparisons/druid-vs-spark.html">Apache Druid vs Spark</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/comparisons/druid-vs-sql-on-hadoop.html">Apache Druid vs SQL-on-Hadoop</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/design/auth.html">Authentication and Authorization</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/design/broker.html">Broker</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/design/coordinator.html">Coordinator Process</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/design/historical.html">Historical Process</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/design/indexer.html">Indexer Process</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/design/indexing-service.html">Indexing Service</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/design/middlemanager.html">MiddleManager Process</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/design/overlord.html">Overlord Process</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/design/router.html">Router Process</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/design/peons.html">Peons</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-core/approximate-histograms.html">Approximate Histogram aggregators</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-core/avro.html">Apache Avro</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-core/azure.html">Microsoft Azure</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-core/bloom-filter.html">Bloom Filter</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-core/datasketches-extension.html">DataSketches extension</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-core/datasketches-hll.html">DataSketches HLL Sketch module</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-core/datasketches-quantiles.html">DataSketches Quantiles Sketch module</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-core/datasketches-theta.html">DataSketches Theta Sketch module</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-core/datasketches-tuple.html">DataSketches Tuple Sketch module</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-core/druid-basic-security.html">Basic Security</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-core/druid-kerberos.html">Kerberos</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-core/druid-lookups.html">Cached Lookup Module</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-core/druid-ranger-security.html">Apache Ranger Security</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-core/google.html">Google Cloud Storage</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-core/hdfs.html">HDFS</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-core/kafka-extraction-namespace.html">Apache Kafka Lookups</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-core/lookups-cached-global.html">Globally Cached Lookups</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-core/mysql.html">MySQL Metadata Store</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-core/orc.html">ORC Extension</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-core/druid-pac4j.html">Druid pac4j based Security extension</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-core/parquet.html">Apache Parquet Extension</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-core/postgresql.html">PostgreSQL Metadata Store</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-core/protobuf.html">Protobuf</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-core/s3.html">S3-compatible</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-core/simple-client-sslcontext.html">Simple SSLContext Provider Module</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-core/stats.html">Stats aggregator</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-core/test-stats.html">Test Stats Aggregators</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-contrib/ambari-metrics-emitter.html">Ambari Metrics Emitter</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-contrib/cassandra.html">Apache Cassandra</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-contrib/cloudfiles.html">Rackspace Cloud Files</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-contrib/distinctcount.html">DistinctCount Aggregator</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-contrib/graphite.html">Graphite Emitter</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-contrib/influx.html">InfluxDB Line Protocol Parser</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-contrib/influxdb-emitter.html">InfluxDB Emitter</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-contrib/kafka-emitter.html">Kafka Emitter</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-contrib/materialized-view.html">Materialized View</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-contrib/momentsketch-quantiles.html">Moment Sketches for Approximate Quantiles module</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-contrib/moving-average-query.html">Moving Average Query</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-contrib/opentsdb-emitter.html">OpenTSDB Emitter</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-contrib/redis-cache.html">Druid Redis Cache</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-contrib/sqlserver.html">Microsoft SQLServer</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-contrib/statsd.html">StatsD Emitter</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-contrib/tdigestsketch-quantiles.html">T-Digest Quantiles Sketch module</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-contrib/thrift.html">Thrift</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-contrib/time-min-max.html">Timestamp Min/Max aggregators</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-contrib/gce-extensions.html">GCE Extensions</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/development/extensions-contrib/aliyun-oss.html">Aliyun OSS</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/querying/hll-old.html">Cardinality/HyperUnique aggregators</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/querying/select-query.html">Select</a></li><li class="navListItem"><a class="navItem" href="/docs/latest/ingestion/standalone-realtime.html">Realtime Process</a></li></ul></div></div></section></div><script> |
| var coll = document.getElementsByClassName('collapsible'); |
| var checkActiveCategory = true; |
| for (var i = 0; i < coll.length; i++) { |
| var links = coll[i].nextElementSibling.getElementsByTagName('*'); |
| if (checkActiveCategory){ |
| for (var j = 0; j < links.length; j++) { |
| if (links[j].classList.contains('navListItemActive')){ |
| coll[i].nextElementSibling.classList.toggle('hide'); |
| coll[i].childNodes[1].classList.toggle('rotate'); |
| checkActiveCategory = false; |
| break; |
| } |
| } |
| } |
| |
| coll[i].addEventListener('click', function() { |
| var arrow = this.childNodes[1]; |
| arrow.classList.toggle('rotate'); |
| var content = this.nextElementSibling; |
| content.classList.toggle('hide'); |
| }); |
| } |
| |
| document.addEventListener('DOMContentLoaded', function() { |
| createToggler('#navToggler', '#docsNav', 'docsSliderActive'); |
| createToggler('#tocToggler', 'body', 'tocActive'); |
| |
| var headings = document.querySelector('.toc-headings'); |
| headings && headings.addEventListener('click', function(event) { |
| var el = event.target; |
| while(el !== headings){ |
| if (el.tagName === 'A') { |
| document.body.classList.remove('tocActive'); |
| break; |
| } else{ |
| el = el.parentNode; |
| } |
| } |
| }, false); |
| |
| function createToggler(togglerSelector, targetSelector, className) { |
| var toggler = document.querySelector(togglerSelector); |
| var target = document.querySelector(targetSelector); |
| |
| if (!toggler) { |
| return; |
| } |
| |
| toggler.onclick = function(event) { |
| event.preventDefault(); |
| |
| target.classList.toggle(className); |
| }; |
| } |
| }); |
| </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/configuration/index.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Configuration reference</h1></header><article><div><span><!-- |
| ~ Licensed to the Apache Software Foundation (ASF) under one |
| ~ or more contributor license agreements. See the NOTICE file |
| ~ distributed with this work for additional information |
| ~ regarding copyright ownership. The ASF licenses this file |
| ~ to you under the Apache License, Version 2.0 (the |
| ~ "License"); you may not use this file except in compliance |
| ~ with the License. You may obtain a copy of the License at |
| ~ |
| ~ http://www.apache.org/licenses/LICENSE-2.0 |
| ~ |
| ~ Unless required by applicable law or agreed to in writing, |
| ~ software distributed under the License is distributed on an |
| ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| ~ KIND, either express or implied. See the License for the |
| ~ specific language governing permissions and limitations |
| ~ under the License. |
| --> |
| <p>This page documents all of the configuration properties for each Druid service type.</p> |
| <h2><a class="anchor" aria-hidden="true" id="recommended-configuration-file-organization"></a><a href="#recommended-configuration-file-organization" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Recommended Configuration File Organization</h2> |
| <p>A recommended way of organizing Druid configuration files can be seen in the <code>conf</code> directory in the Druid package root, shown below:</p> |
| <pre><code class="hljs">$ <span class="hljs-keyword">ls</span> -R <span class="hljs-keyword">conf</span> |
| druid |
| |
| <span class="hljs-keyword">conf</span>/druid: |
| _common broker coordinator historical middleManager overlord |
| |
| <span class="hljs-keyword">conf</span>/druid/_common: |
| common.<span class="hljs-keyword">runtime</span>.properties log4j2.xml |
| |
| <span class="hljs-keyword">conf</span>/druid/broker: |
| jvm.config <span class="hljs-keyword">runtime</span>.properties |
| |
| <span class="hljs-keyword">conf</span>/druid/coordinator: |
| jvm.config <span class="hljs-keyword">runtime</span>.properties |
| |
| <span class="hljs-keyword">conf</span>/druid/historica<span class="hljs-variable">l:</span> |
| jvm.config <span class="hljs-keyword">runtime</span>.properties |
| |
| <span class="hljs-keyword">conf</span>/druid/middleManager: |
| jvm.config <span class="hljs-keyword">runtime</span>.properties |
| |
| <span class="hljs-keyword">conf</span>/druid/overlord: |
| jvm.config <span class="hljs-keyword">runtime</span>.properties |
| </code></pre> |
| <p>Each directory has a <code>runtime.properties</code> file containing configuration properties for the specific Druid process corresponding to the directory (e.g., <code>historical</code>).</p> |
| <p>The <code>jvm.config</code> files contain JVM flags such as heap sizing properties for each service.</p> |
| <p>Common properties shared by all services are placed in <code>_common/common.runtime.properties</code>.</p> |
| <h2><a class="anchor" aria-hidden="true" id="common-configurations"></a><a href="#common-configurations" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Common Configurations</h2> |
| <p>The properties under this section are common configurations that should be shared across all Druid services in a cluster.</p> |
| <h3><a class="anchor" aria-hidden="true" id="jvm-configuration-best-practices"></a><a href="#jvm-configuration-best-practices" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>JVM Configuration Best Practices</h3> |
| <p>There are four JVM parameters that we set on all of our processes:</p> |
| <ol> |
| <li><code>-Duser.timezone=UTC</code> This sets the default timezone of the JVM to UTC. We always set this and do not test with other default timezones, so local timezones might work, but they also might uncover weird and interesting bugs. To issue queries in a non-UTC timezone, see <a href="../querying/granularities.html#period-granularities">query granularities</a></li> |
| <li><code>-Dfile.encoding=UTF-8</code> This is similar to timezone, we test assuming UTF-8. Local encodings might work, but they also might result in weird and interesting bugs.</li> |
| <li><code>-Djava.io.tmpdir=<a path></code> Various parts of the system that interact with the file system do it via temporary files, and these files can get somewhat large. Many production systems are set up to have small (but fast) <code>/tmp</code> directories, which can be problematic with Druid so we recommend pointing the JVM’s tmp directory to something with a little more meat. This directory should not be volatile tmpfs. This directory should also have good read and write speed and hence NFS mount should strongly be avoided.</li> |
| <li><code>-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager</code> This allows log4j2 to handle logs for non-log4j2 components (like jetty) which use standard java logging.</li> |
| </ol> |
| <h3><a class="anchor" aria-hidden="true" id="extensions"></a><a href="#extensions" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Extensions</h3> |
| <p>Many of Druid's external dependencies can be plugged in as modules. Extensions can be provided using the following configs:</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.extensions.directory</code></td><td>The root extension directory where user can put extensions related files. Druid will load extensions stored under this directory.</td><td><code>extensions</code> (This is a relative path to Druid's working directory)</td></tr> |
| <tr><td><code>druid.extensions.hadoopDependenciesDir</code></td><td>The root hadoop dependencies directory where user can put hadoop related dependencies files. Druid will load the dependencies based on the hadoop coordinate specified in the hadoop index task.</td><td><code>hadoop-dependencies</code> (This is a relative path to Druid's working directory</td></tr> |
| <tr><td><code>druid.extensions.loadList</code></td><td>A JSON array of extensions to load from extension directories by Druid. If it is not specified, its value will be <code>null</code> and Druid will load all the extensions under <code>druid.extensions.directory</code>. If its value is empty list <code>[]</code>, then no extensions will be loaded at all. It is also allowed to specify absolute path of other custom extensions not stored in the common extensions directory.</td><td>null</td></tr> |
| <tr><td><code>druid.extensions.searchCurrentClassloader</code></td><td>This is a boolean flag that determines if Druid will search the main classloader for extensions. It defaults to true but can be turned off if you have reason to not automatically add all modules on the classpath.</td><td>true</td></tr> |
| <tr><td><code>druid.extensions.useExtensionClassloaderFirst</code></td><td>This is a boolean flag that determines if Druid extensions should prefer loading classes from their own jars rather than jars bundled with Druid. If false, extensions must be compatible with classes provided by any jars bundled with Druid. If true, extensions may depend on conflicting versions.</td><td>false</td></tr> |
| <tr><td><code>druid.extensions.hadoopContainerDruidClasspath</code></td><td>Hadoop Indexing launches hadoop jobs and this configuration provides way to explicitly set the user classpath for the hadoop job. By default this is computed automatically by druid based on the druid process classpath and set of extensions. However, sometimes you might want to be explicit to resolve dependency conflicts between druid and hadoop.</td><td>null</td></tr> |
| <tr><td><code>druid.extensions.addExtensionsToHadoopContainer</code></td><td>Only applicable if <code>druid.extensions.hadoopContainerDruidClasspath</code> is provided. If set to true, then extensions specified in the loadList are added to hadoop container classpath. Note that when <code>druid.extensions.hadoopContainerDruidClasspath</code> is not provided then extensions are always added to hadoop container classpath.</td><td>false</td></tr> |
| </tbody> |
| </table> |
| <h3><a class="anchor" aria-hidden="true" id="modules"></a><a href="#modules" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Modules</h3> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.modules.excludeList</code></td><td>A JSON array of canonical class names (e.g., <code>"org.apache.druid.somepackage.SomeModule"</code>) of module classes which shouldn't be loaded, even if they are found in extensions specified by <code>druid.extensions.loadList</code>, or in the list of core modules specified to be loaded on a particular Druid process type. Useful when some useful extension contains some module, which shouldn't be loaded on some Druid process type because some dependencies of that module couldn't be satisfied.</td><td>[]</td></tr> |
| </tbody> |
| </table> |
| <h3><a class="anchor" aria-hidden="true" id="zookeeper"></a><a href="#zookeeper" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Zookeeper</h3> |
| <p>We recommend just setting the base ZK path and the ZK service host, but all ZK paths that Druid uses can be overwritten to absolute paths.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.zk.paths.base</code></td><td>Base Zookeeper path.</td><td><code>/druid</code></td></tr> |
| <tr><td><code>druid.zk.service.host</code></td><td>The ZooKeeper hosts to connect to. This is a REQUIRED property and therefore a host address must be supplied.</td><td>none</td></tr> |
| <tr><td><code>druid.zk.service.user</code></td><td>The username to authenticate with ZooKeeper. This is an optional property.</td><td>none</td></tr> |
| <tr><td><code>druid.zk.service.pwd</code></td><td>The <a href="/docs/latest/operations/password-provider.html">Password Provider</a> or the string password to authenticate with ZooKeeper. This is an optional property.</td><td>none</td></tr> |
| <tr><td><code>druid.zk.service.authScheme</code></td><td>digest is the only authentication scheme supported.</td><td>digest</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="zookeeper-behavior"></a><a href="#zookeeper-behavior" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Zookeeper Behavior</h4> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.zk.service.sessionTimeoutMs</code></td><td>ZooKeeper session timeout, in milliseconds.</td><td><code>30000</code></td></tr> |
| <tr><td><code>druid.zk.service.connectionTimeoutMs</code></td><td>ZooKeeper connection timeout, in milliseconds.</td><td><code>15000</code></td></tr> |
| <tr><td><code>druid.zk.service.compress</code></td><td>Boolean flag for whether or not created Znodes should be compressed.</td><td><code>true</code></td></tr> |
| <tr><td><code>druid.zk.service.acl</code></td><td>Boolean flag for whether or not to enable ACL security for ZooKeeper. If ACL is enabled, zNode creators will have all permissions.</td><td><code>false</code></td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="path-configuration"></a><a href="#path-configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Path Configuration</h4> |
| <p>Druid interacts with ZK through a set of standard path configurations. We recommend just setting the base ZK path, but all ZK paths that Druid uses can be overwritten to absolute paths.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.zk.paths.base</code></td><td>Base Zookeeper path.</td><td><code>/druid</code></td></tr> |
| <tr><td><code>druid.zk.paths.propertiesPath</code></td><td>Zookeeper properties path.</td><td><code>${druid.zk.paths.base}/properties</code></td></tr> |
| <tr><td><code>druid.zk.paths.announcementsPath</code></td><td>Druid process announcement path.</td><td><code>${druid.zk.paths.base}/announcements</code></td></tr> |
| <tr><td><code>druid.zk.paths.liveSegmentsPath</code></td><td>Current path for where Druid processes announce their segments.</td><td><code>${druid.zk.paths.base}/segments</code></td></tr> |
| <tr><td><code>druid.zk.paths.loadQueuePath</code></td><td>Entries here cause Historical processes to load and drop segments.</td><td><code>${druid.zk.paths.base}/loadQueue</code></td></tr> |
| <tr><td><code>druid.zk.paths.coordinatorPath</code></td><td>Used by the Coordinator for leader election.</td><td><code>${druid.zk.paths.base}/coordinator</code></td></tr> |
| <tr><td><code>druid.zk.paths.servedSegmentsPath</code></td><td>@Deprecated. Legacy path for where Druid processes announce their segments.</td><td><code>${druid.zk.paths.base}/servedSegments</code></td></tr> |
| </tbody> |
| </table> |
| <p>The indexing service also uses its own set of paths. These configs can be included in the common configuration.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.zk.paths.indexer.base</code></td><td>Base zookeeper path for</td><td><code>${druid.zk.paths.base}/indexer</code></td></tr> |
| <tr><td><code>druid.zk.paths.indexer.announcementsPath</code></td><td>Middle managers announce themselves here.</td><td><code>${druid.zk.paths.indexer.base}/announcements</code></td></tr> |
| <tr><td><code>druid.zk.paths.indexer.tasksPath</code></td><td>Used to assign tasks to MiddleManagers.</td><td><code>${druid.zk.paths.indexer.base}/tasks</code></td></tr> |
| <tr><td><code>druid.zk.paths.indexer.statusPath</code></td><td>Parent path for announcement of task statuses.</td><td><code>${druid.zk.paths.indexer.base}/status</code></td></tr> |
| </tbody> |
| </table> |
| <p>If <code>druid.zk.paths.base</code> and <code>druid.zk.paths.indexer.base</code> are both set, and none of the other <code>druid.zk.paths.*</code> or <code>druid.zk.paths.indexer.*</code> values are set, then the other properties will be evaluated relative to their respective <code>base</code>. |
| For example, if <code>druid.zk.paths.base</code> is set to <code>/druid1</code> and <code>druid.zk.paths.indexer.base</code> is set to <code>/druid2</code> then <code>druid.zk.paths.announcementsPath</code> will default to <code>/druid1/announcements</code> while <code>druid.zk.paths.indexer.announcementsPath</code> will default to <code>/druid2/announcements</code>.</p> |
| <p>The following path is used for service discovery. It is <strong>not</strong> affected by <code>druid.zk.paths.base</code> and <strong>must</strong> be specified separately.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.discovery.curator.path</code></td><td>Services announce themselves under this ZooKeeper path.</td><td><code>/druid/discovery</code></td></tr> |
| </tbody> |
| </table> |
| <h3><a class="anchor" aria-hidden="true" id="exhibitor"></a><a href="#exhibitor" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Exhibitor</h3> |
| <p><a href="https://github.com/Netflix/exhibitor/wiki">Exhibitor</a> is a supervisor system for ZooKeeper. |
| Exhibitor can dynamically scale-up/down the cluster of ZooKeeper servers. |
| Druid can update self-owned list of ZooKeeper servers through Exhibitor without restarting. |
| That is, it allows Druid to keep the connections of Exhibitor-supervised ZooKeeper servers.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.exhibitor.service.hosts</code></td><td>A JSON array which contains the hostnames of Exhibitor instances. Please specify this property if you want to use Exhibitor-supervised cluster.</td><td>none</td></tr> |
| <tr><td><code>druid.exhibitor.service.port</code></td><td>The REST port used to connect to Exhibitor.</td><td><code>8080</code></td></tr> |
| <tr><td><code>druid.exhibitor.service.restUriPath</code></td><td>The path of the REST call used to get the server set.</td><td><code>/exhibitor/v1/cluster/list</code></td></tr> |
| <tr><td><code>druid.exhibitor.service.useSsl</code></td><td>Boolean flag for whether or not to use https protocol.</td><td><code>false</code></td></tr> |
| <tr><td><code>druid.exhibitor.service.pollingMs</code></td><td>How often to poll the exhibitors for the list</td><td><code>10000</code></td></tr> |
| </tbody> |
| </table> |
| <p>Note that <code>druid.zk.service.host</code> is used as a backup in case an Exhibitor instance can't be contacted and therefore should still be set.</p> |
| <h3><a class="anchor" aria-hidden="true" id="tls"></a><a href="#tls" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>TLS</h3> |
| <h4><a class="anchor" aria-hidden="true" id="general-configuration"></a><a href="#general-configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>General Configuration</h4> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.enablePlaintextPort</code></td><td>Enable/Disable HTTP connector.</td><td><code>true</code></td></tr> |
| <tr><td><code>druid.enableTlsPort</code></td><td>Enable/Disable HTTPS connector.</td><td><code>false</code></td></tr> |
| </tbody> |
| </table> |
| <p>Although not recommended but both HTTP and HTTPS connectors can be enabled at a time and respective ports are configurable using <code>druid.plaintextPort</code> |
| and <code>druid.tlsPort</code> properties on each process. Please see <code>Configuration</code> section of individual processes to check the valid and default values for these ports.</p> |
| <h4><a class="anchor" aria-hidden="true" id="jetty-server-tls-configuration"></a><a href="#jetty-server-tls-configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Jetty Server TLS Configuration</h4> |
| <p>Druid uses Jetty as an embedded web server. To get familiar with TLS/SSL in general and related concepts like Certificates etc. |
| reading this <a href="http://www.eclipse.org/jetty/documentation/9.4.x/configuring-ssl.html">Jetty documentation</a> might be helpful. |
| To get more in depth knowledge of TLS/SSL support in Java in general, please refer to this <a href="http://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html">guide</a>. |
| The documentation <a href="http://www.eclipse.org/jetty/documentation/9.4.x/configuring-ssl.html#configuring-sslcontextfactory">here</a> |
| can help in understanding TLS/SSL configurations listed below. This <a href="http://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html">document</a> lists all the possible |
| values for the below mentioned configs among others provided by Java implementation.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th><th>Required</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.server.https.keyStorePath</code></td><td>The file path or URL of the TLS/SSL Key store.</td><td>none</td><td>yes</td></tr> |
| <tr><td><code>druid.server.https.keyStoreType</code></td><td>The type of the key store.</td><td>none</td><td>yes</td></tr> |
| <tr><td><code>druid.server.https.certAlias</code></td><td>Alias of TLS/SSL certificate for the connector.</td><td>none</td><td>yes</td></tr> |
| <tr><td><code>druid.server.https.keyStorePassword</code></td><td>The <a href="/docs/latest/operations/password-provider.html">Password Provider</a> or String password for the Key Store.</td><td>none</td><td>yes</td></tr> |
| </tbody> |
| </table> |
| <p>Following table contains non-mandatory advanced configuration options, use caution.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th><th>Required</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.server.https.keyManagerFactoryAlgorithm</code></td><td>Algorithm to use for creating KeyManager, more details <a href="https://docs.oracle.com/javase/7/docs/technotes/guides/security/jsse/JSSERefGuide.html#KeyManager">here</a>.</td><td><code>javax.net.ssl.KeyManagerFactory.getDefaultAlgorithm()</code></td><td>no</td></tr> |
| <tr><td><code>druid.server.https.keyManagerPassword</code></td><td>The <a href="/docs/latest/operations/password-provider.html">Password Provider</a> or String password for the Key Manager.</td><td>none</td><td>no</td></tr> |
| <tr><td><code>druid.server.https.includeCipherSuites</code></td><td>List of cipher suite names to include. You can either use the exact cipher suite name or a regular expression.</td><td>Jetty's default include cipher list</td><td>no</td></tr> |
| <tr><td><code>druid.server.https.excludeCipherSuites</code></td><td>List of cipher suite names to exclude. You can either use the exact cipher suite name or a regular expression.</td><td>Jetty's default exclude cipher list</td><td>no</td></tr> |
| <tr><td><code>druid.server.https.includeProtocols</code></td><td>List of exact protocols names to include.</td><td>Jetty's default include protocol list</td><td>no</td></tr> |
| <tr><td><code>druid.server.https.excludeProtocols</code></td><td>List of exact protocols names to exclude.</td><td>Jetty's default exclude protocol list</td><td>no</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="internal-client-tls-configuration-requires-simple-client-sslcontext-extension"></a><a href="#internal-client-tls-configuration-requires-simple-client-sslcontext-extension" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Internal Client TLS Configuration (requires <code>simple-client-sslcontext</code> extension)</h4> |
| <p>These properties apply to the SSLContext that will be provided to the internal HTTP client that Druid services use to communicate with each other. These properties require the <code>simple-client-sslcontext</code> extension to be loaded. Without it, Druid services will be unable to communicate with each other when TLS is enabled.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th><th>Required</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.client.https.protocol</code></td><td>SSL protocol to use.</td><td><code>TLSv1.2</code></td><td>no</td></tr> |
| <tr><td><code>druid.client.https.trustStoreType</code></td><td>The type of the key store where trusted root certificates are stored.</td><td><code>java.security.KeyStore.getDefaultType()</code></td><td>no</td></tr> |
| <tr><td><code>druid.client.https.trustStorePath</code></td><td>The file path or URL of the TLS/SSL Key store where trusted root certificates are stored.</td><td>none</td><td>yes</td></tr> |
| <tr><td><code>druid.client.https.trustStoreAlgorithm</code></td><td>Algorithm to be used by TrustManager to validate certificate chains</td><td><code>javax.net.ssl.TrustManagerFactory.getDefaultAlgorithm()</code></td><td>no</td></tr> |
| <tr><td><code>druid.client.https.trustStorePassword</code></td><td>The <a href="/docs/latest/operations/password-provider.html">Password Provider</a> or String password for the Trust Store.</td><td>none</td><td>yes</td></tr> |
| </tbody> |
| </table> |
| <p>This <a href="http://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html">document</a> lists all the possible |
| values for the above mentioned configs among others provided by Java implementation.</p> |
| <h3><a class="anchor" aria-hidden="true" id="authentication-and-authorization"></a><a href="#authentication-and-authorization" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Authentication and Authorization</h3> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Type</th><th>Description</th><th>Default</th><th>Required</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.auth.authenticatorChain</code></td><td>JSON List of Strings</td><td>List of Authenticator type names</td><td>["allowAll"]</td><td>no</td></tr> |
| <tr><td><code>druid.escalator.type</code></td><td>String</td><td>Type of the Escalator that should be used for internal Druid communications. This Escalator must use an authentication scheme that is supported by an Authenticator in <code>druid.auth.authenticatorChain</code>.</td><td>"noop"</td><td>no</td></tr> |
| <tr><td><code>druid.auth.authorizers</code></td><td>JSON List of Strings</td><td>List of Authorizer type names</td><td>["allowAll"]</td><td>no</td></tr> |
| <tr><td><code>druid.auth.unsecuredPaths</code></td><td>List of Strings</td><td>List of paths for which security checks will not be performed. All requests to these paths will be allowed.</td><td>[]</td><td>no</td></tr> |
| <tr><td><code>druid.auth.allowUnauthenticatedHttpOptions</code></td><td>Boolean</td><td>If true, skip authentication checks for HTTP OPTIONS requests. This is needed for certain use cases, such as supporting CORS pre-flight requests. Note that disabling authentication checks for OPTIONS requests will allow unauthenticated users to determine what Druid endpoints are valid (by checking if the OPTIONS request returns a 200 instead of 404), so enabling this option may reveal information about server configuration, including information about what extensions are loaded (if those extensions add endpoints).</td><td>false</td><td>no</td></tr> |
| </tbody> |
| </table> |
| <p>For more information, please see <a href="/docs/latest/design/auth.html">Authentication and Authorization</a>.</p> |
| <p>For configuration options for specific auth extensions, please refer to the extension documentation.</p> |
| <h3><a class="anchor" aria-hidden="true" id="startup-logging"></a><a href="#startup-logging" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Startup Logging</h3> |
| <p>All processes can log debugging information on startup.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.startup.logging.logProperties</code></td><td>Log all properties on startup (from common.runtime.properties, runtime.properties, and the JVM command line).</td><td>false</td></tr> |
| <tr><td><code>druid.startup.logging.maskProperties</code></td><td>Masks sensitive properties (passwords, for example) containing theses words.</td><td>["password"]</td></tr> |
| </tbody> |
| </table> |
| <p>Note that some sensitive information may be logged if these settings are enabled.</p> |
| <h3><a class="anchor" aria-hidden="true" id="request-logging"></a><a href="#request-logging" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Request Logging</h3> |
| <p>All processes that can serve queries can also log the query requests they see. Broker processes can additionally log the SQL requests (both from HTTP and JDBC) they see.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.request.logging.type</code></td><td>Choices: noop, file, emitter, slf4j, filtered, composing, switching. How to log every query request.</td><td>[required to configure request logging]</td></tr> |
| </tbody> |
| </table> |
| <p>Note that, you can enable sending all the HTTP requests to log by setting "org.apache.druid.jetty.RequestLog" to DEBUG level. See <a href="/docs/latest/configuration/logging.html">Logging</a></p> |
| <h4><a class="anchor" aria-hidden="true" id="file-request-logging"></a><a href="#file-request-logging" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>File Request Logging</h4> |
| <p>Daily request logs are stored on disk.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.request.logging.dir</code></td><td>Historical, Realtime and Broker processes maintain request logs of all of the requests they get (interaction is via POST, so normal request logs don’t generally capture information about the actual query), this specifies the directory to store the request logs in</td><td>none</td></tr> |
| <tr><td><code>druid.request.logging.filePattern</code></td><td><a href="http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html">Joda datetime format</a> for each file</td><td>"yyyy-MM-dd'.log'"</td></tr> |
| </tbody> |
| </table> |
| <p>The format of request logs is TSV, one line per requests, with five fields: timestamp, remote_addr, native_query, query_context, sql_query.</p> |
| <p>For native JSON request, the <code>sql_query</code> field is empty. Example</p> |
| <pre><code class="hljs"><span class="hljs-number">2019-01</span><span class="hljs-number">-14T10</span><span class="hljs-symbol">:</span>00<span class="hljs-symbol">:</span>00.000Z <span class="hljs-number">127.0</span>.0.<span class="hljs-number">1</span> {<span class="hljs-string">"queryType"</span><span class="hljs-symbol">:<span class="hljs-string">"topN"</span></span>,<span class="hljs-string">"dataSource"</span><span class="hljs-symbol">:</span>{<span class="hljs-string">"type"</span><span class="hljs-symbol">:<span class="hljs-string">"table"</span></span>,<span class="hljs-string">"name"</span><span class="hljs-symbol">:<span class="hljs-string">"wikiticker"</span></span>},<span class="hljs-string">"virtualColumns"</span><span class="hljs-symbol">:[]</span>,<span class="hljs-string">"dimension"</span><span class="hljs-symbol">:</span>{<span class="hljs-string">"type"</span><span class="hljs-symbol">:<span class="hljs-string">"LegacyDimensionSpec"</span></span>,<span class="hljs-string">"dimension"</span><span class="hljs-symbol">:<span class="hljs-string">"page"</span></span>,<span class="hljs-string">"outputName"</span><span class="hljs-symbol">:<span class="hljs-string">"page"</span></span>,<span class="hljs-string">"outputType"</span><span class="hljs-symbol">:<span class="hljs-string">"STRING"</span></span>},<span class="hljs-string">"metric"</span><span class="hljs-symbol">:</span>{<span class="hljs-string">"type"</span><span class="hljs-symbol">:<span class="hljs-string">"LegacyTopNMetricSpec"</span></span>,<span class="hljs-string">"metric"</span><span class="hljs-symbol">:<span class="hljs-string">"count"</span></span>},<span class="hljs-string">"threshold"</span><span class="hljs-symbol">:</span><span class="hljs-number">10</span>,<span class="hljs-string">"intervals"</span><span class="hljs-symbol">:</span>{<span class="hljs-string">"type"</span><span class="hljs-symbol">:<span class="hljs-string">"LegacySegmentSpec"</span></span>,<span class="hljs-string">"intervals"</span><span class="hljs-symbol">:</span>[<span class="hljs-string">"2015-09-12T00:00:00.000Z/2015-09-13T00:00:00.000Z"</span>]},<span class="hljs-string">"filter"</span><span class="hljs-symbol">:null</span>,<span class="hljs-string">"granularity"</span><span class="hljs-symbol">:</span>{<span class="hljs-string">"type"</span><span class="hljs-symbol">:<span class="hljs-string">"all"</span></span>},<span class="hljs-string">"aggregations"</span><span class="hljs-symbol">:</span>[{<span class="hljs-string">"type"</span><span class="hljs-symbol">:<span class="hljs-string">"count"</span></span>,<span class="hljs-string">"name"</span><span class="hljs-symbol">:<span class="hljs-string">"count"</span></span>}],<span class="hljs-string">"postAggregations"</span><span class="hljs-symbol">:[]</span>,<span class="hljs-string">"context"</span><span class="hljs-symbol">:</span>{<span class="hljs-string">"queryId"</span><span class="hljs-symbol">:<span class="hljs-string">"74c2d540-d700-4ebd-b4a9-3d02397976aa"</span></span>},<span class="hljs-string">"descending"</span><span class="hljs-symbol">:false</span>} {<span class="hljs-string">"query/time"</span><span class="hljs-symbol">:</span><span class="hljs-number">100</span>,<span class="hljs-string">"query/bytes"</span><span class="hljs-symbol">:</span><span class="hljs-number">800</span>,<span class="hljs-string">"success"</span><span class="hljs-symbol">:true</span>,<span class="hljs-string">"identity"</span><span class="hljs-symbol">:<span class="hljs-string">"user1"</span></span>} |
| </code></pre> |
| <p>For SQL query request, the <code>native_query</code> field is empty. Example</p> |
| <pre><code class="hljs"><span class="hljs-number">2019</span><span class="hljs-number">-01</span><span class="hljs-number">-14</span><span class="hljs-string">T10:</span><span class="hljs-number">00</span>:<span class="hljs-number">00.000</span>Z <span class="hljs-number">127.0</span><span class="hljs-number">.0</span><span class="hljs-number">.1</span> {<span class="hljs-string">"sqlQuery/time"</span>:<span class="hljs-number">100</span>,<span class="hljs-string">"sqlQuery/bytes"</span>:<span class="hljs-number">600</span>,<span class="hljs-string">"success"</span>:<span class="hljs-literal">true</span>,<span class="hljs-string">"identity"</span>:<span class="hljs-string">"user1"</span>} {<span class="hljs-string">"query"</span>:<span class="hljs-string">"SELECT page, COUNT(*) AS Edits FROM wikiticker WHERE __time BETWEEN TIMESTAMP '2015-09-12 00:00:00' AND TIMESTAMP '2015-09-13 00:00:00' GROUP BY page ORDER BY Edits DESC LIMIT 10"</span>,<span class="hljs-string">"context"</span>:{<span class="hljs-string">"sqlQueryId"</span>:<span class="hljs-string">"c9d035a0-5ffd-4a79-a865-3ffdadbb5fdd"</span>,<span class="hljs-string">"nativeQueryIds"</span>:<span class="hljs-string">"[490978e4-f5c7-4cf6-b174-346e63cf8863]"</span>}} |
| </code></pre> |
| <h4><a class="anchor" aria-hidden="true" id="emitter-request-logging"></a><a href="#emitter-request-logging" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Emitter Request Logging</h4> |
| <p>Every request is emitted to some external location.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.request.logging.feed</code></td><td>Feed name for requests.</td><td>none</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="slf4j-request-logging"></a><a href="#slf4j-request-logging" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>SLF4J Request Logging</h4> |
| <p>Every request is logged via SLF4J. Native queries are serialized into JSON in the log message regardless of the SLF4J format specification. They will be logged under the class <code>org.apache.druid.server.log.LoggingRequestLogger</code>.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.request.logging.setMDC</code></td><td>If MDC entries should be set in the log entry. Your logging setup still has to be configured to handle MDC to format this data</td><td>false</td></tr> |
| <tr><td><code>druid.request.logging.setContextMDC</code></td><td>If the druid query <code>context</code> should be added to the MDC entries. Has no effect unless <code>setMDC</code> is <code>true</code></td><td>false</td></tr> |
| </tbody> |
| </table> |
| <p>For native query, the following MDC fields are populated with <code>setMDC</code>:</p> |
| <table> |
| <thead> |
| <tr><th>MDC field</th><th>Description</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>queryId</code></td><td>The query ID</td></tr> |
| <tr><td><code>sqlQueryId</code></td><td>The SQL query ID if this query is part of a SQL request</td></tr> |
| <tr><td><code>dataSource</code></td><td>The datasource the query was against</td></tr> |
| <tr><td><code>queryType</code></td><td>The type of the query</td></tr> |
| <tr><td><code>hasFilters</code></td><td>If the query has any filters</td></tr> |
| <tr><td><code>remoteAddr</code></td><td>The remote address of the requesting client</td></tr> |
| <tr><td><code>duration</code></td><td>The duration of the query interval</td></tr> |
| <tr><td><code>resultOrdering</code></td><td>The ordering of results</td></tr> |
| <tr><td><code>descending</code></td><td>If the query is a descending query</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="filtered-request-logging"></a><a href="#filtered-request-logging" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Filtered Request Logging</h4> |
| <p>Filtered Request Logger filters requests based on a configurable query/time threshold (for native query) and sqlQuery/time threshold (for SQL query). |
| For native query, only request logs where query/time is above the threshold are emitted. For SQL query, only request logs where sqlQuery/time is above the threshold are emitted.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.request.logging.queryTimeThresholdMs</code></td><td>Threshold value for query/time in milliseconds.</td><td>0, i.e., no filtering</td></tr> |
| <tr><td><code>druid.request.logging.sqlQueryTimeThresholdMs</code></td><td>Threshold value for sqlQuery/time in milliseconds.</td><td>0, i.e., no filtering</td></tr> |
| <tr><td><code>druid.request.logging.mutedQueryTypes</code></td><td>Query requests of these types are not logged. Query types are defined as string objects corresponding to the "queryType" value for the specified query in the Druid's <a href="http://druid.apache.org/docs/latest/querying/querying.html">native JSON query API</a>. Misspelled query types will be ignored. Example to ignore scan and timeBoundary queries: ["scan", "timeBoundary"]</td><td>[]</td></tr> |
| <tr><td><code>druid.request.logging.delegate.type</code></td><td>Type of delegate request logger to log requests.</td><td>none</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="composite-request-logging"></a><a href="#composite-request-logging" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Composite Request Logging</h4> |
| <p>Composite Request Logger emits request logs to multiple request loggers.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.request.logging.loggerProviders</code></td><td>List of request loggers for emitting request logs.</td><td>none</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="switching-request-logging"></a><a href="#switching-request-logging" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Switching Request Logging</h4> |
| <p>Switching Request Logger routes native query's request logs to one request logger and SQL query's request logs to another request logger.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.request.logging.nativeQueryLogger</code></td><td>request logger for emitting native query's request logs.</td><td>none</td></tr> |
| <tr><td><code>druid.request.logging.sqlQueryLogger</code></td><td>request logger for emitting SQL query's request logs.</td><td>none</td></tr> |
| </tbody> |
| </table> |
| <h3><a class="anchor" aria-hidden="true" id="audit-logging"></a><a href="#audit-logging" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Audit Logging</h3> |
| <p>Coordinator and Overlord log changes to lookups, segment load/drop rules, dynamic configuration changes for auditing</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.audit.manager.auditHistoryMillis</code></td><td>Default duration for querying audit history.</td><td>1 week</td></tr> |
| <tr><td><code>druid.audit.manager.includePayloadAsDimensionInMetric</code></td><td>Boolean flag on whether to add <code>payload</code> column in service metric.</td><td>false</td></tr> |
| </tbody> |
| </table> |
| <h3><a class="anchor" aria-hidden="true" id="enabling-metrics"></a><a href="#enabling-metrics" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Enabling Metrics</h3> |
| <p>Druid processes periodically emit metrics and different metrics monitors can be included. Each process can overwrite the default list of monitors.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.monitoring.emissionPeriod</code></td><td>How often metrics are emitted.</td><td>PT1M</td></tr> |
| <tr><td><code>druid.monitoring.monitors</code></td><td>Sets list of Druid monitors used by a process. See below for names and more information. For example, you can specify monitors for a Broker with <code>druid.monitoring.monitors=["org.apache.druid.java.util.metrics.SysMonitor","org.apache.druid.java.util.metrics.JvmMonitor"]</code>.</td><td>none (no monitors)</td></tr> |
| </tbody> |
| </table> |
| <p>The following monitors are available:</p> |
| <table> |
| <thead> |
| <tr><th>Name</th><th>Description</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>org.apache.druid.client.cache.CacheMonitor</code></td><td>Emits metrics (to logs) about the segment results cache for Historical and Broker processes. Reports typical cache statistics include hits, misses, rates, and size (bytes and number of entries), as well as timeouts and and errors.</td></tr> |
| <tr><td><code>org.apache.druid.java.util.metrics.SysMonitor</code></td><td>This uses the <a href="https://github.com/hyperic/sigar">SIGAR library</a> to report on various system activities and statuses.</td></tr> |
| <tr><td><code>org.apache.druid.server.metrics.HistoricalMetricsMonitor</code></td><td>Reports statistics on Historical processes.</td></tr> |
| <tr><td><code>org.apache.druid.java.util.metrics.JvmMonitor</code></td><td>Reports various JVM-related statistics.</td></tr> |
| <tr><td><code>org.apache.druid.java.util.metrics.JvmCpuMonitor</code></td><td>Reports statistics of CPU consumption by the JVM.</td></tr> |
| <tr><td><code>org.apache.druid.java.util.metrics.CpuAcctDeltaMonitor</code></td><td>Reports consumed CPU as per the cpuacct cgroup.</td></tr> |
| <tr><td><code>org.apache.druid.java.util.metrics.JvmThreadsMonitor</code></td><td>Reports Thread statistics in the JVM, like numbers of total, daemon, started, died threads.</td></tr> |
| <tr><td><code>org.apache.druid.segment.realtime.RealtimeMetricsMonitor</code></td><td>Reports statistics on Realtime processes.</td></tr> |
| <tr><td><code>org.apache.druid.server.metrics.EventReceiverFirehoseMonitor</code></td><td>Reports how many events have been queued in the EventReceiverFirehose.</td></tr> |
| <tr><td><code>org.apache.druid.server.metrics.QueryCountStatsMonitor</code></td><td>Reports how many queries have been successful/failed/interrupted.</td></tr> |
| <tr><td><code>org.apache.druid.server.emitter.HttpEmittingMonitor</code></td><td>Reports internal metrics of <code>http</code> or <code>parametrized</code> emitter (see below). Must not be used with another emitter type. See the description of the metrics here: <a href="https://github.com/apache/druid/pull/4973">https://github.com/apache/druid/pull/4973</a>.</td></tr> |
| <tr><td><code>org.apache.druid.server.metrics.TaskCountStatsMonitor</code></td><td>Reports how many ingestion tasks are currently running/pending/waiting and also the number of successful/failed tasks per emission period.</td></tr> |
| </tbody> |
| </table> |
| <h3><a class="anchor" aria-hidden="true" id="emitting-metrics"></a><a href="#emitting-metrics" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Emitting Metrics</h3> |
| <p>The Druid servers <a href="/docs/latest/operations/metrics.html">emit various metrics</a> and alerts via something we call an Emitter. There are three emitter implementations included with the code, a "noop" emitter (the default if none is specified), one that just logs to log4j ("logging"), and one that does POSTs of JSON events to a server ("http"). The properties for using the logging emitter are described below.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.emitter</code></td><td>Setting this value to "noop", "logging", "http" or "parametrized" will initialize one of the emitter modules. The value "composing" can be used to initialize multiple emitter modules.</td><td>noop</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="logging-emitter-module"></a><a href="#logging-emitter-module" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Logging Emitter Module</h4> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.emitter.logging.loggerClass</code></td><td>Choices: HttpPostEmitter, LoggingEmitter, NoopServiceEmitter, ServiceEmitter. The class used for logging.</td><td>LoggingEmitter</td></tr> |
| <tr><td><code>druid.emitter.logging.logLevel</code></td><td>Choices: debug, info, warn, error. The log level at which message are logged.</td><td>info</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="http-emitter-module"></a><a href="#http-emitter-module" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Http Emitter Module</h4> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.emitter.http.flushMillis</code></td><td>How often the internal message buffer is flushed (data is sent).</td><td>60000</td></tr> |
| <tr><td><code>druid.emitter.http.flushCount</code></td><td>How many messages the internal message buffer can hold before flushing (sending).</td><td>500</td></tr> |
| <tr><td><code>druid.emitter.http.basicAuthentication</code></td><td><a href="/docs/latest/operations/password-provider.html">Password Provider</a> for providing Login and password for authentication in "login:password" form, e.g., <code>druid.emitter.http.basicAuthentication=admin:adminpassword</code> uses Default Password Provider which allows plain text passwords.</td><td>not specified = no authentication</td></tr> |
| <tr><td><code>druid.emitter.http.flushTimeOut</code></td><td>The timeout after which an event should be sent to the endpoint, even if internal buffers are not filled, in milliseconds.</td><td>not specified = no timeout</td></tr> |
| <tr><td><code>druid.emitter.http.batchingStrategy</code></td><td>The strategy of how the batch is formatted. "ARRAY" means <code>[event1,event2]</code>, "NEWLINES" means <code>event1\nevent2</code>, ONLY_EVENTS means <code>event1event2</code>.</td><td>ARRAY</td></tr> |
| <tr><td><code>druid.emitter.http.maxBatchSize</code></td><td>The maximum batch size, in bytes.</td><td>the minimum of (10% of JVM heap size divided by 2) or (5191680 (i. e. 5 MB))</td></tr> |
| <tr><td><code>druid.emitter.http.batchQueueSizeLimit</code></td><td>The maximum number of batches in emitter queue, if there are problems with emitting.</td><td>the maximum of (2) or (10% of the JVM heap size divided by 5MB)</td></tr> |
| <tr><td><code>druid.emitter.http.minHttpTimeoutMillis</code></td><td>If the speed of filling batches imposes timeout smaller than that, not even trying to send batch to endpoint, because it will likely fail, not being able to send the data that fast. Configure this depending based on emitter/successfulSending/minTimeMs metric. Reasonable values are 10ms..100ms.</td><td>0</td></tr> |
| <tr><td><code>druid.emitter.http.recipientBaseUrl</code></td><td>The base URL to emit messages to. Druid will POST JSON to be consumed at the HTTP endpoint specified by this property.</td><td>none, required config</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="http-emitter-module-tls-overrides"></a><a href="#http-emitter-module-tls-overrides" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Http Emitter Module TLS Overrides</h4> |
| <p>When emitting events to a TLS-enabled receiver, the Http Emitter will by default use an SSLContext obtained via the |
| process described at <a href="../operations/tls-support.html">Druid's internal communication over TLS</a>, i.e., the same |
| SSLContext that would be used for internal communications between Druid processes.</p> |
| <p>In some use cases it may be desirable to have the Http Emitter use its own separate truststore configuration. For example, there may be organizational policies that prevent the TLS-enabled metrics receiver's certificate from being added to the same truststore used by Druid's internal HTTP client.</p> |
| <p>The following properties allow the Http Emitter to use its own truststore configuration when building its SSLContext.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.emitter.http.ssl.useDefaultJavaContext</code></td><td>If set to true, the HttpEmitter will use <code>SSLContext.getDefault()</code>, the default Java SSLContext, and all other properties below are ignored.</td><td>false</td></tr> |
| <tr><td><code>druid.emitter.http.ssl.trustStorePath</code></td><td>The file path or URL of the TLS/SSL Key store where trusted root certificates are stored. If this is unspecified, the Http Emitter will use the same SSLContext as Druid's internal HTTP client, as described in the beginning of this section, and all other properties below are ignored.</td><td>null</td></tr> |
| <tr><td><code>druid.emitter.http.ssl.trustStoreType</code></td><td>The type of the key store where trusted root certificates are stored.</td><td><code>java.security.KeyStore.getDefaultType()</code></td></tr> |
| <tr><td><code>druid.emitter.http.ssl.trustStoreAlgorithm</code></td><td>Algorithm to be used by TrustManager to validate certificate chains</td><td><code>javax.net.ssl.TrustManagerFactory.getDefaultAlgorithm()</code></td></tr> |
| <tr><td><code>druid.emitter.http.ssl.trustStorePassword</code></td><td>The <a href="/docs/latest/operations/password-provider.html">Password Provider</a> or String password for the Trust Store.</td><td>none</td></tr> |
| <tr><td><code>druid.emitter.http.ssl.protocol</code></td><td>TLS protocol to use.</td><td>"TLSv1.2"</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="parametrized-http-emitter-module"></a><a href="#parametrized-http-emitter-module" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Parametrized Http Emitter Module</h4> |
| <p><code>druid.emitter.parametrized.httpEmitting.*</code> configs correspond to the configs of Http Emitter Modules, see above. |
| Except <code>recipientBaseUrl</code>. E.g., <code>druid.emitter.parametrized.httpEmitting.flushMillis</code>, |
| <code>druid.emitter.parametrized.httpEmitting.flushCount</code>, <code>druid.emitter.parametrized.httpEmitting.ssl.trustStorePath</code>, etc.</p> |
| <p>The additional configs are:</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.emitter.parametrized.recipientBaseUrlPattern</code></td><td>The URL pattern to send an event to, based on the event's feed. E.g., <code>http://foo.bar/{feed}</code>, that will send event to <code>http://foo.bar/metrics</code> if the event's feed is "metrics".</td><td>none, required config</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="composing-emitter-module"></a><a href="#composing-emitter-module" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Composing Emitter Module</h4> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.emitter.composing.emitters</code></td><td>List of emitter modules to load, e.g., ["logging","http"].</td><td>[]</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="graphite-emitter"></a><a href="#graphite-emitter" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Graphite Emitter</h4> |
| <p>To use graphite as emitter set <code>druid.emitter=graphite</code>. For configuration details please follow this <a href="/docs/latest/development/extensions-contrib/graphite.html">link</a>.</p> |
| <h3><a class="anchor" aria-hidden="true" id="metadata-storage"></a><a href="#metadata-storage" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Metadata storage</h3> |
| <p>These properties specify the JDBC connection and other configuration around the metadata storage. The only processes that connect to the metadata storage with these properties are the <a href="/docs/latest/design/coordinator.html">Coordinator</a> and <a href="/docs/latest/design/overlord.html">Overlord</a>.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.metadata.storage.type</code></td><td>The type of metadata storage to use. Choose from "mysql", "postgresql", or "derby".</td><td>derby</td></tr> |
| <tr><td><code>druid.metadata.storage.connector.connectURI</code></td><td>The JDBC URI for the database to connect to</td><td>none</td></tr> |
| <tr><td><code>druid.metadata.storage.connector.user</code></td><td>The username to connect with.</td><td>none</td></tr> |
| <tr><td><code>druid.metadata.storage.connector.password</code></td><td>The <a href="/docs/latest/operations/password-provider.html">Password Provider</a> or String password used to connect with.</td><td>none</td></tr> |
| <tr><td><code>druid.metadata.storage.connector.createTables</code></td><td>If Druid requires a table and it doesn't exist, create it?</td><td>true</td></tr> |
| <tr><td><code>druid.metadata.storage.tables.base</code></td><td>The base name for tables.</td><td>druid</td></tr> |
| <tr><td><code>druid.metadata.storage.tables.dataSource</code></td><td>The table to use to look for dataSources which created by <a href="/docs/latest/development/extensions-core/kafka-ingestion.html">Kafka Indexing Service</a>.</td><td>druid_dataSource</td></tr> |
| <tr><td><code>druid.metadata.storage.tables.pendingSegments</code></td><td>The table to use to look for pending segments.</td><td>druid_pendingSegments</td></tr> |
| <tr><td><code>druid.metadata.storage.tables.segments</code></td><td>The table to use to look for segments.</td><td>druid_segments</td></tr> |
| <tr><td><code>druid.metadata.storage.tables.rules</code></td><td>The table to use to look for segment load/drop rules.</td><td>druid_rules</td></tr> |
| <tr><td><code>druid.metadata.storage.tables.config</code></td><td>The table to use to look for configs.</td><td>druid_config</td></tr> |
| <tr><td><code>druid.metadata.storage.tables.tasks</code></td><td>Used by the indexing service to store tasks.</td><td>druid_tasks</td></tr> |
| <tr><td><code>druid.metadata.storage.tables.taskLog</code></td><td>Used by the indexing service to store task logs.</td><td>druid_taskLog</td></tr> |
| <tr><td><code>druid.metadata.storage.tables.taskLock</code></td><td>Used by the indexing service to store task locks.</td><td>druid_taskLock</td></tr> |
| <tr><td><code>druid.metadata.storage.tables.supervisors</code></td><td>Used by the indexing service to store supervisor configurations.</td><td>druid_supervisors</td></tr> |
| <tr><td><code>druid.metadata.storage.tables.audit</code></td><td>The table to use for audit history of configuration changes, e.g., Coordinator rules.</td><td>druid_audit</td></tr> |
| </tbody> |
| </table> |
| <h3><a class="anchor" aria-hidden="true" id="deep-storage"></a><a href="#deep-storage" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Deep storage</h3> |
| <p>The configurations concern how to push and pull <a href="/docs/latest/design/segments.html">Segments</a> from deep storage.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.storage.type</code></td><td>Choices:local, noop, s3, hdfs, c*. The type of deep storage to use.</td><td>local</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="local-deep-storage"></a><a href="#local-deep-storage" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Local Deep Storage</h4> |
| <p>Local deep storage uses the local filesystem.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.storage.storageDirectory</code></td><td>Directory on disk to use as deep storage.</td><td>/tmp/druid/localStorage</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="noop-deep-storage"></a><a href="#noop-deep-storage" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Noop Deep Storage</h4> |
| <p>This deep storage doesn't do anything. There are no configs.</p> |
| <h4><a class="anchor" aria-hidden="true" id="s3-deep-storage"></a><a href="#s3-deep-storage" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>S3 Deep Storage</h4> |
| <p>This deep storage is used to interface with Amazon's S3. Note that the <code>druid-s3-extensions</code> extension must be loaded. |
| The below table shows some important configurations for S3. See <a href="/docs/latest/development/extensions-core/s3.html">S3 Deep Storage</a> for full configurations.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.storage.bucket</code></td><td>S3 bucket name.</td><td>none</td></tr> |
| <tr><td><code>druid.storage.baseKey</code></td><td>S3 object key prefix for storage.</td><td>none</td></tr> |
| <tr><td><code>druid.storage.disableAcl</code></td><td>Boolean flag for ACL. If this is set to <code>false</code>, the full control would be granted to the bucket owner. This may require to set additional permissions. See <a href="../development/extensions-core/s3.html#s3-permissions-settings">S3 permissions settings</a>.</td><td>false</td></tr> |
| <tr><td><code>druid.storage.archiveBucket</code></td><td>S3 bucket name for archiving when running the <em>archive task</em>.</td><td>none</td></tr> |
| <tr><td><code>druid.storage.archiveBaseKey</code></td><td>S3 object key prefix for archiving.</td><td>none</td></tr> |
| <tr><td><code>druid.storage.sse.type</code></td><td>Server-side encryption type. Should be one of <code>s3</code>, <code>kms</code>, and <code>custom</code>. See the below <a href="../development/extensions-core/s3.html#server-side-encryption">Server-side encryption section</a> for more details.</td><td>None</td></tr> |
| <tr><td><code>druid.storage.sse.kms.keyId</code></td><td>AWS KMS key ID. This is used only when <code>druid.storage.sse.type</code> is <code>kms</code> and can be empty to use the default key ID.</td><td>None</td></tr> |
| <tr><td><code>druid.storage.sse.custom.base64EncodedKey</code></td><td>Base64-encoded key. Should be specified if <code>druid.storage.sse.type</code> is <code>custom</code>.</td><td>None</td></tr> |
| <tr><td><code>druid.storage.useS3aSchema</code></td><td>If true, use the "s3a" filesystem when using Hadoop-based ingestion. If false, the "s3n" filesystem will be used. Only affects Hadoop-based ingestion.</td><td>false</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="hdfs-deep-storage"></a><a href="#hdfs-deep-storage" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>HDFS Deep Storage</h4> |
| <p>This deep storage is used to interface with HDFS. Note that the <code>druid-hdfs-storage</code> extension must be loaded.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.storage.storageDirectory</code></td><td>HDFS directory to use as deep storage.</td><td>none</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="cassandra-deep-storage"></a><a href="#cassandra-deep-storage" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Cassandra Deep Storage</h4> |
| <p>This deep storage is used to interface with Cassandra. Note that the <code>druid-cassandra-storage</code> extension must be loaded.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.storage.host</code></td><td>Cassandra host.</td><td>none</td></tr> |
| <tr><td><code>druid.storage.keyspace</code></td><td>Cassandra key space.</td><td>none</td></tr> |
| </tbody> |
| </table> |
| <h3><a class="anchor" aria-hidden="true" id="task-logging"></a><a href="#task-logging" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Task Logging</h3> |
| <p>If you are running the indexing service in remote mode, the task logs must be stored in S3, Azure Blob Store, Google Cloud Storage or HDFS.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.indexer.logs.type</code></td><td>Choices:noop, s3, azure, google, hdfs, file. Where to store task logs</td><td>file</td></tr> |
| </tbody> |
| </table> |
| <p>You can also configure the Overlord to automatically retain the task logs in log directory and entries in task-related metadata storage tables only for last x milliseconds by configuring following additional properties. |
| Caution: Automatic log file deletion typically works based on log file modification timestamp on the backing store, so large clock skews between druid processes and backing store nodes might result in unintended behavior.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.indexer.logs.kill.enabled</code></td><td>Boolean value for whether to enable deletion of old task logs. If set to true, Overlord will submit kill tasks periodically based on <code>druid.indexer.logs.kill.delay</code> specified, which will delete task logs from the log directory as well as tasks and tasklogs table entries in metadata storage except for tasks created in the last <code>druid.indexer.logs.kill.durationToRetain</code> period.</td><td>false</td></tr> |
| <tr><td><code>druid.indexer.logs.kill.durationToRetain</code></td><td>Required if kill is enabled. In milliseconds, task logs and entries in task-related metadata storage tables to be retained created in last x milliseconds.</td><td>None</td></tr> |
| <tr><td><code>druid.indexer.logs.kill.initialDelay</code></td><td>Optional. Number of milliseconds after Overlord start when first auto kill is run.</td><td>random value less than 300000 (5 mins)</td></tr> |
| <tr><td><code>druid.indexer.logs.kill.delay</code></td><td>Optional. Number of milliseconds of delay between successive executions of auto kill run.</td><td>21600000 (6 hours)</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="file-task-logs"></a><a href="#file-task-logs" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>File Task Logs</h4> |
| <p>Store task logs in the local filesystem.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.indexer.logs.directory</code></td><td>Local filesystem path.</td><td>log</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="s3-task-logs"></a><a href="#s3-task-logs" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>S3 Task Logs</h4> |
| <p>Store task logs in S3. Note that the <code>druid-s3-extensions</code> extension must be loaded.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.indexer.logs.s3Bucket</code></td><td>S3 bucket name.</td><td>none</td></tr> |
| <tr><td><code>druid.indexer.logs.s3Prefix</code></td><td>S3 key prefix.</td><td>none</td></tr> |
| <tr><td><code>druid.indexer.logs.disableAcl</code></td><td>Boolean flag for ACL. If this is set to <code>false</code>, the full control would be granted to the bucket owner. If the task logs bucket is the same as the deep storage (S3) bucket, then the value of this property will need to be set to true if druid.storage.disableAcl has been set to true.</td><td>false</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="azure-blob-store-task-logs"></a><a href="#azure-blob-store-task-logs" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Azure Blob Store Task Logs</h4> |
| <p>Store task logs in Azure Blob Store.</p> |
| <p>Note: The <code>druid-azure-extensions</code> extension must be loaded, and this uses the same storage account as the deep storage module for azure.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.indexer.logs.container</code></td><td>The Azure Blob Store container to write logs to</td><td>none</td></tr> |
| <tr><td><code>druid.indexer.logs.prefix</code></td><td>The path to prepend to logs</td><td>none</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="google-cloud-storage-task-logs"></a><a href="#google-cloud-storage-task-logs" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Google Cloud Storage Task Logs</h4> |
| <p>Store task logs in Google Cloud Storage.</p> |
| <p>Note: The <code>druid-google-extensions</code> extension must be loaded, and this uses the same storage settings as the deep storage module for google.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.indexer.logs.bucket</code></td><td>The Google Cloud Storage bucket to write logs to</td><td>none</td></tr> |
| <tr><td><code>druid.indexer.logs.prefix</code></td><td>The path to prepend to logs</td><td>none</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="hdfs-task-logs"></a><a href="#hdfs-task-logs" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>HDFS Task Logs</h4> |
| <p>Store task logs in HDFS. Note that the <code>druid-hdfs-storage</code> extension must be loaded.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.indexer.logs.directory</code></td><td>The directory to store logs.</td><td>none</td></tr> |
| </tbody> |
| </table> |
| <h3><a class="anchor" aria-hidden="true" id="overlord-discovery"></a><a href="#overlord-discovery" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Overlord Discovery</h3> |
| <p>This config is used to find the <a href="/docs/latest/design/overlord.html">Overlord</a> using Curator service discovery. Only required if you are actually running an Overlord.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.selectors.indexing.serviceName</code></td><td>The druid.service name of the Overlord process. To start the Overlord with a different name, set it with this property.</td><td>druid/overlord</td></tr> |
| </tbody> |
| </table> |
| <h3><a class="anchor" aria-hidden="true" id="coordinator-discovery"></a><a href="#coordinator-discovery" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Coordinator Discovery</h3> |
| <p>This config is used to find the <a href="/docs/latest/design/coordinator.html">Coordinator</a> using Curator service discovery. This config is used by the realtime indexing processes to get information about the segments loaded in the cluster.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.selectors.coordinator.serviceName</code></td><td>The druid.service name of the Coordinator process. To start the Coordinator with a different name, set it with this property.</td><td>druid/coordinator</td></tr> |
| </tbody> |
| </table> |
| <h3><a class="anchor" aria-hidden="true" id="announcing-segments"></a><a href="#announcing-segments" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Announcing Segments</h3> |
| <p>You can configure how to announce and unannounce Znodes in ZooKeeper (using Curator). For normal operations you do not need to override any of these configs.</p> |
| <h5><a class="anchor" aria-hidden="true" id="batch-data-segment-announcer"></a><a href="#batch-data-segment-announcer" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Batch Data Segment Announcer</h5> |
| <p>In current Druid, multiple data segments may be announced under the same Znode.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.announcer.segmentsPerNode</code></td><td>Each Znode contains info for up to this many segments.</td><td>50</td></tr> |
| <tr><td><code>druid.announcer.maxBytesPerNode</code></td><td>Max byte size for Znode.</td><td>524288</td></tr> |
| <tr><td><code>druid.announcer.skipDimensionsAndMetrics</code></td><td>Skip Dimensions and Metrics list from segment announcements. NOTE: Enabling this will also remove the dimensions and metrics list from Coordinator and Broker endpoints.</td><td>false</td></tr> |
| <tr><td><code>druid.announcer.skipLoadSpec</code></td><td>Skip segment LoadSpec from segment announcements. NOTE: Enabling this will also remove the loadspec from Coordinator and Broker endpoints.</td><td>false</td></tr> |
| </tbody> |
| </table> |
| <h3><a class="anchor" aria-hidden="true" id="javascript"></a><a href="#javascript" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>JavaScript</h3> |
| <p>Druid supports dynamic runtime extension through JavaScript functions. This functionality can be configured through |
| the following properties.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.javascript.enabled</code></td><td>Set to "true" to enable JavaScript functionality. This affects the JavaScript parser, filter, extractionFn, aggregator, post-aggregator, router strategy, and worker selection strategy.</td><td>false</td></tr> |
| </tbody> |
| </table> |
| <blockquote> |
| <p>JavaScript-based functionality is disabled by default. Please refer to the Druid <a href="/docs/latest/development/javascript.html">JavaScript programming guide</a> for guidelines about using Druid's JavaScript functionality, including instructions on how to enable it.</p> |
| </blockquote> |
| <h3><a class="anchor" aria-hidden="true" id="double-column-storage"></a><a href="#double-column-storage" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Double Column storage</h3> |
| <p>Prior to version 0.13.0, Druid's storage layer used a 32-bit float representation to store columns created by the |
| doubleSum, doubleMin, and doubleMax aggregators at indexing time. |
| Starting from version 0.13.0 the default will be 64-bit floats for Double columns. |
| Using 64-bit representation for double column will lead to avoid precision loss at the cost of doubling the storage size of such columns. |
| To keep the old format set the system-wide property <code>druid.indexing.doubleStorage=float</code>. |
| You can also use floatSum, floatMin and floatMax to use 32-bit float representation. |
| Support for 64-bit floating point columns was released in Druid 0.11.0, so if you use this feature then older versions of Druid will not be able to read your data segments.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.indexing.doubleStorage</code></td><td>Set to "float" to use 32-bit double representation for double columns.</td><td>double</td></tr> |
| </tbody> |
| </table> |
| <h3><a class="anchor" aria-hidden="true" id="sql-compatible-null-handling"></a><a href="#sql-compatible-null-handling" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>SQL compatible null handling</h3> |
| <p>Prior to version 0.13.0, Druid string columns treated <code>''</code> and <code>null</code> values as interchangeable, and numeric columns were unable to represent <code>null</code> values, coercing <code>null</code> to <code>0</code>. Druid 0.13.0 introduced a mode which enabled SQL compatible null handling, allowing string columns to distinguish empty strings from nulls, and numeric columns to contain null rows.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.generic.useDefaultValueForNull</code></td><td>When set to <code>true</code>, <code>null</code> values will be stored as <code>''</code> for string columns and <code>0</code> for numeric columns. Set to <code>false</code> to store and query data in SQL compatible mode.</td><td><code>true</code></td></tr> |
| </tbody> |
| </table> |
| <p>This mode does have a storage size and query performance cost, see <a href="/docs/latest/design/segments.html#sql-compatible-null-handling">segment documentation</a> for more details.</p> |
| <h3><a class="anchor" aria-hidden="true" id="http-client"></a><a href="#http-client" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>HTTP Client</h3> |
| <p>All Druid components can communicate with each other over HTTP.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.global.http.numConnections</code></td><td>Size of connection pool per destination URL. If there are more HTTP requests than this number that all need to speak to the same URL, then they will queue up.</td><td><code>20</code></td></tr> |
| <tr><td><code>druid.global.http.compressionCodec</code></td><td>Compression codec to communicate with others. May be "gzip" or "identity".</td><td><code>gzip</code></td></tr> |
| <tr><td><code>druid.global.http.readTimeout</code></td><td>The timeout for data reads.</td><td><code>PT15M</code></td></tr> |
| <tr><td><code>druid.global.http.unusedConnectionTimeout</code></td><td>The timeout for idle connections in connection pool. The connection in the pool will be closed after this timeout and a new one will be established. This timeout should be less than <code>druid.global.http.readTimeout</code>. Set this timeout = ~90% of <code>druid.global.http.readTimeout</code></td><td><code>PT4M</code></td></tr> |
| <tr><td><code>druid.global.http.numMaxThreads</code></td><td>Maximum number of I/O worker threads</td><td><code>max(10, ((number of cores * 17) / 16 + 2) + 30)</code></td></tr> |
| </tbody> |
| </table> |
| <h2><a class="anchor" aria-hidden="true" id="master-server"></a><a href="#master-server" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Master Server</h2> |
| <p>This section contains the configuration options for the processes that reside on Master servers (Coordinators and Overlords) in the suggested <a href="../design/processes.html#server-types">three-server configuration</a>.</p> |
| <h3><a class="anchor" aria-hidden="true" id="coordinator"></a><a href="#coordinator" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Coordinator</h3> |
| <p>For general Coordinator Process information, see <a href="/docs/latest/design/coordinator.html">here</a>.</p> |
| <h4><a class="anchor" aria-hidden="true" id="static-configuration"></a><a href="#static-configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Static Configuration</h4> |
| <p>These Coordinator static configurations can be defined in the <code>coordinator/runtime.properties</code> file.</p> |
| <h5><a class="anchor" aria-hidden="true" id="coordinator-process-config"></a><a href="#coordinator-process-config" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Coordinator Process Config</h5> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.host</code></td><td>The host for the current process. This is used to advertise the current processes location as reachable from another process and should generally be specified such that <code>http://${druid.host}/</code> could actually talk to this process</td><td>InetAddress.getLocalHost().getCanonicalHostName()</td></tr> |
| <tr><td><code>druid.bindOnHost</code></td><td>Indicating whether the process's internal jetty server bind on <code>druid.host</code>. Default is false, which means binding to all interfaces.</td><td>false</td></tr> |
| <tr><td><code>druid.plaintextPort</code></td><td>This is the port to actually listen on; unless port mapping is used, this will be the same port as is on <code>druid.host</code></td><td>8081</td></tr> |
| <tr><td><code>druid.tlsPort</code></td><td>TLS port for HTTPS connector, if <a href="/docs/latest/operations/tls-support.html">druid.enableTlsPort</a> is set then this config will be used. If <code>druid.host</code> contains port then that port will be ignored. This should be a non-negative Integer.</td><td>8281</td></tr> |
| <tr><td><code>druid.service</code></td><td>The name of the service. This is used as a dimension when emitting metrics and alerts to differentiate between the various services</td><td>druid/coordinator</td></tr> |
| </tbody> |
| </table> |
| <h5><a class="anchor" aria-hidden="true" id="coordinator-operation"></a><a href="#coordinator-operation" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Coordinator Operation</h5> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.coordinator.period</code></td><td>The run period for the Coordinator. The Coordinator operates by maintaining the current state of the world in memory and periodically looking at the set of "used" segments and segments being served to make decisions about whether any changes need to be made to the data topology. This property sets the delay between each of these runs.</td><td>PT60S</td></tr> |
| <tr><td><code>druid.coordinator.period.indexingPeriod</code></td><td>How often to send compact/merge/conversion tasks to the indexing service. It's recommended to be longer than <code>druid.manager.segments.pollDuration</code></td><td>PT1800S (30 mins)</td></tr> |
| <tr><td><code>druid.coordinator.startDelay</code></td><td>The operation of the Coordinator works on the assumption that it has an up-to-date view of the state of the world when it runs, the current ZK interaction code, however, is written in a way that doesn’t allow the Coordinator to know for a fact that it’s done loading the current state of the world. This delay is a hack to give it enough time to believe that it has all the data.</td><td>PT300S</td></tr> |
| <tr><td><code>druid.coordinator.load.timeout</code></td><td>The timeout duration for when the Coordinator assigns a segment to a Historical process.</td><td>PT15M</td></tr> |
| <tr><td><code>druid.coordinator.kill.pendingSegments.on</code></td><td>Boolean flag for whether or not the Coordinator clean up old entries in the <code>pendingSegments</code> table of metadata store. If set to true, Coordinator will check the created time of most recently complete task. If it doesn't exist, it finds the created time of the earliest running/pending/waiting tasks. Once the created time is found, then for all dataSources not in the <code>killPendingSegmentsSkipList</code> (see <a href="#dynamic-configuration">Dynamic configuration</a>), Coordinator will ask the Overlord to clean up the entries 1 day or more older than the found created time in the <code>pendingSegments</code> table. This will be done periodically based on <code>druid.coordinator.period</code> specified.</td><td>true</td></tr> |
| <tr><td><code>druid.coordinator.kill.on</code></td><td>Boolean flag for whether or not the Coordinator should submit kill task for unused segments, that is, hard delete them from metadata store and deep storage. If set to true, then for all whitelisted dataSources (or optionally all), Coordinator will submit tasks periodically based on <code>period</code> specified. These kill tasks will delete all unused segments except for the last <code>durationToRetain</code> period. Whitelist or All can be set via dynamic configuration <code>killAllDataSources</code> and <code>killDataSourceWhitelist</code> described later.</td><td>false</td></tr> |
| <tr><td><code>druid.coordinator.kill.period</code></td><td>How often to send kill tasks to the indexing service. Value must be greater than <code>druid.coordinator.period.indexingPeriod</code>. Only applies if kill is turned on.</td><td>P1D (1 Day)</td></tr> |
| <tr><td><code>druid.coordinator.kill.durationToRetain</code></td><td>Do not kill unused segments in last <code>durationToRetain</code>, must be greater or equal to 0. Only applies and MUST be specified if kill is turned on. Note that default value is invalid.</td><td>PT-1S (-1 seconds)</td></tr> |
| <tr><td><code>druid.coordinator.kill.maxSegments</code></td><td>Kill at most n unused segments per kill task submission, must be greater than 0. Only applies and MUST be specified if kill is turned on. Note that default value is invalid.</td><td>0</td></tr> |
| <tr><td><code>druid.coordinator.balancer.strategy</code></td><td>Specify the type of balancing strategy that the coordinator should use to distribute segments among the historicals. <code>cachingCost</code> is logically equivalent to <code>cost</code> but is more CPU-efficient on large clusters and will replace <code>cost</code> in the future versions, users are invited to try it. Use <code>diskNormalized</code> to distribute segments among processes so that the disks fill up uniformly and use <code>random</code> to randomly pick processes to distribute segments.</td><td><code>cost</code></td></tr> |
| <tr><td><code>druid.coordinator.balancer.cachingCost.awaitInitialization</code></td><td>Whether to wait for segment view initialization before creating the <code>cachingCost</code> balancing strategy. This property is enabled only when <code>druid.coordinator.balancer.strategy</code> is <code>cachingCost</code>. If set to 'true', the Coordinator will not start to assign segments, until the segment view is initialized. If set to 'false', the Coordinator will fallback to use the <code>cost</code> balancing strategy only if the segment view is not initialized yet. Notes, it may take much time to wait for the initialization since the <code>cachingCost</code> balancing strategy involves much computing to build itself.</td><td>false</td></tr> |
| <tr><td><code>druid.coordinator.loadqueuepeon.repeatDelay</code></td><td>The start and repeat delay for the loadqueuepeon, which manages the load and drop of segments.</td><td>PT0.050S (50 ms)</td></tr> |
| <tr><td><code>druid.coordinator.asOverlord.enabled</code></td><td>Boolean value for whether this Coordinator process should act like an Overlord as well. This configuration allows users to simplify a druid cluster by not having to deploy any standalone Overlord processes. If set to true, then Overlord console is available at <code>http://coordinator-host:port/console.html</code> and be sure to set <code>druid.coordinator.asOverlord.overlordService</code> also. See next.</td><td>false</td></tr> |
| <tr><td><code>druid.coordinator.asOverlord.overlordService</code></td><td>Required, if <code>druid.coordinator.asOverlord.enabled</code> is <code>true</code>. This must be same value as <code>druid.service</code> on standalone Overlord processes and <code>druid.selectors.indexing.serviceName</code> on Middle Managers.</td><td>NULL</td></tr> |
| </tbody> |
| </table> |
| <h5><a class="anchor" aria-hidden="true" id="segment-management"></a><a href="#segment-management" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Segment Management</h5> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Possible Values</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.serverview.type</code></td><td>batch or http</td><td>Segment discovery method to use. "http" enables discovering segments using HTTP instead of zookeeper.</td><td>batch</td></tr> |
| <tr><td><code>druid.coordinator.loadqueuepeon.type</code></td><td>curator or http</td><td>Whether to use "http" or "curator" implementation to assign segment loads/drops to historical</td><td>curator</td></tr> |
| <tr><td><code>druid.coordinator.segment.awaitInitializationOnStart</code></td><td>true or false</td><td>Whether the Coordinator will wait for its view of segments to fully initialize before starting up. If set to 'true', the Coordinator's HTTP server will not start up, and the Coordinator will not announce itself as available, until the server view is initialized.</td><td>true</td></tr> |
| </tbody> |
| </table> |
| <h6><a class="anchor" aria-hidden="true" id="additional-config-when-http-loadqueuepeon-is-used"></a><a href="#additional-config-when-http-loadqueuepeon-is-used" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Additional config when "http" loadqueuepeon is used</h6> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.coordinator.loadqueuepeon.http.batchSize</code></td><td>Number of segment load/drop requests to batch in one HTTP request. Note that it must be smaller than <code>druid.segmentCache.numLoadingThreads</code> config on Historical process.</td><td>1</td></tr> |
| </tbody> |
| </table> |
| <h5><a class="anchor" aria-hidden="true" id="metadata-retrieval"></a><a href="#metadata-retrieval" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Metadata Retrieval</h5> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.manager.config.pollDuration</code></td><td>How often the manager polls the config table for updates.</td><td>PT1M</td></tr> |
| <tr><td><code>druid.manager.segments.pollDuration</code></td><td>The duration between polls the Coordinator does for updates to the set of active segments. Generally defines the amount of lag time it can take for the Coordinator to notice new segments.</td><td>PT1M</td></tr> |
| <tr><td><code>druid.manager.rules.pollDuration</code></td><td>The duration between polls the Coordinator does for updates to the set of active rules. Generally defines the amount of lag time it can take for the Coordinator to notice rules.</td><td>PT1M</td></tr> |
| <tr><td><code>druid.manager.rules.defaultRule</code></td><td>The default rule for the cluster</td><td>_default</td></tr> |
| <tr><td><code>druid.manager.rules.alertThreshold</code></td><td>The duration after a failed poll upon which an alert should be emitted.</td><td>PT10M</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="dynamic-configuration"></a><a href="#dynamic-configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Dynamic Configuration</h4> |
| <p>The Coordinator has dynamic configuration to change certain behavior on the fly. The Coordinator uses a JSON spec object from the Druid <a href="/docs/latest/dependencies/metadata-storage.html">metadata storage</a> config table. This object is detailed below:</p> |
| <p>It is recommended that you use the Coordinator Console to configure these parameters. However, if you need to do it via HTTP, the JSON object can be submitted to the Coordinator via a POST request at:</p> |
| <pre><code class="hljs">http:<span class="hljs-regexp">//</span><COORDINATOR_IP>:<PORT><span class="hljs-regexp">/druid/</span>coordinator<span class="hljs-regexp">/v1/</span>config |
| </code></pre> |
| <p>Optional Header Parameters for auditing the config change can also be specified.</p> |
| <table> |
| <thead> |
| <tr><th>Header Param Name</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>X-Druid-Author</code></td><td>author making the config change</td><td>""</td></tr> |
| <tr><td><code>X-Druid-Comment</code></td><td>comment describing the change being done</td><td>""</td></tr> |
| </tbody> |
| </table> |
| <p>A sample Coordinator dynamic config JSON object is shown below:</p> |
| <pre><code class="hljs css language-json">{ |
| <span class="hljs-attr">"millisToWaitBeforeDeleting"</span>: <span class="hljs-number">900000</span>, |
| <span class="hljs-attr">"mergeBytesLimit"</span>: <span class="hljs-number">100000000</span>, |
| <span class="hljs-attr">"mergeSegmentsLimit"</span> : <span class="hljs-number">1000</span>, |
| <span class="hljs-attr">"maxSegmentsToMove"</span>: <span class="hljs-number">5</span>, |
| <span class="hljs-attr">"replicantLifetime"</span>: <span class="hljs-number">15</span>, |
| <span class="hljs-attr">"replicationThrottleLimit"</span>: <span class="hljs-number">10</span>, |
| <span class="hljs-attr">"emitBalancingStats"</span>: <span class="hljs-literal">false</span>, |
| <span class="hljs-attr">"killDataSourceWhitelist"</span>: [<span class="hljs-string">"wikipedia"</span>, <span class="hljs-string">"testDatasource"</span>], |
| <span class="hljs-attr">"decommissioningNodes"</span>: [<span class="hljs-string">"localhost:8182"</span>, <span class="hljs-string">"localhost:8282"</span>], |
| <span class="hljs-attr">"decommissioningMaxPercentOfMaxSegmentsToMove"</span>: <span class="hljs-number">70</span>, |
| <span class="hljs-attr">"pauseCoordination"</span>: <span class="hljs-literal">false</span> |
| } |
| </code></pre> |
| <p>Issuing a GET request at the same URL will return the spec that is currently in place. A description of the config setup spec is shown below.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>millisToWaitBeforeDeleting</code></td><td>How long does the Coordinator need to be a leader before it can start marking overshadowed segments as unused in metadata storage.</td><td>900000 (15 mins)</td></tr> |
| <tr><td><code>mergeBytesLimit</code></td><td>The maximum total uncompressed size in bytes of segments to merge.</td><td>524288000L</td></tr> |
| <tr><td><code>mergeSegmentsLimit</code></td><td>The maximum number of segments that can be in a single <a href="/docs/latest/ingestion/tasks.html">append task</a>.</td><td>100</td></tr> |
| <tr><td><code>maxSegmentsToMove</code></td><td>The maximum number of segments that can be moved at any given time.</td><td>5</td></tr> |
| <tr><td><code>replicantLifetime</code></td><td>The maximum number of Coordinator runs for a segment to be replicated before we start alerting.</td><td>15</td></tr> |
| <tr><td><code>replicationThrottleLimit</code></td><td>The maximum number of segments that can be replicated at one time.</td><td>10</td></tr> |
| <tr><td><code>balancerComputeThreads</code></td><td>Thread pool size for computing moving cost of segments in segment balancing. Consider increasing this if you have a lot of segments and moving segments starts to get stuck.</td><td>1</td></tr> |
| <tr><td><code>emitBalancingStats</code></td><td>Boolean flag for whether or not we should emit balancing stats. This is an expensive operation.</td><td>false</td></tr> |
| <tr><td><code>killDataSourceWhitelist</code></td><td>List of specific data sources for which kill tasks are sent if property <code>druid.coordinator.kill.on</code> is true. This can be a list of comma-separated data source names or a JSON array.</td><td>none</td></tr> |
| <tr><td><code>killAllDataSources</code></td><td>Send kill tasks for ALL dataSources if property <code>druid.coordinator.kill.on</code> is true. If this is set to true then <code>killDataSourceWhitelist</code> must not be specified or be empty list.</td><td>false</td></tr> |
| <tr><td><code>killPendingSegmentsSkipList</code></td><td>List of data sources for which pendingSegments are <em>NOT</em> cleaned up if property <code>druid.coordinator.kill.pendingSegments.on</code> is true. This can be a list of comma-separated data sources or a JSON array.</td><td>none</td></tr> |
| <tr><td><code>maxSegmentsInNodeLoadingQueue</code></td><td>The maximum number of segments that could be queued for loading to any given server. This parameter could be used to speed up segments loading process, especially if there are "slow" nodes in the cluster (with low loading speed) or if too much segments scheduled to be replicated to some particular node (faster loading could be preferred to better segments distribution). Desired value depends on segments loading speed, acceptable replication time and number of nodes. Value 1000 could be a start point for a rather big cluster. Default value is 0 (loading queue is unbounded)</td><td>0</td></tr> |
| <tr><td><code>decommissioningNodes</code></td><td>List of historical servers to 'decommission'. Coordinator will not assign new segments to 'decommissioning' servers, and segments will be moved away from them to be placed on non-decommissioning servers at the maximum rate specified by <code>decommissioningMaxPercentOfMaxSegmentsToMove</code>.</td><td>none</td></tr> |
| <tr><td><code>decommissioningMaxPercentOfMaxSegmentsToMove</code></td><td>The maximum number of segments that may be moved away from 'decommissioning' servers to non-decommissioning (that is, active) servers during one Coordinator run. This value is relative to the total maximum segment movements allowed during one run which is determined by <code>maxSegmentsToMove</code>. If <code>decommissioningMaxPercentOfMaxSegmentsToMove</code> is 0, segments will neither be moved from <em>or to</em> 'decommissioning' servers, effectively putting them in a sort of "maintenance" mode that will not participate in balancing or assignment by load rules. Decommissioning can also become stalled if there are no available active servers to place the segments. By leveraging the maximum percent of decommissioning segment movements, an operator can prevent active servers from overload by prioritizing balancing, or decrease decommissioning time instead. The value should be between 0 and 100.</td><td>70</td></tr> |
| <tr><td><code>pauseCoordination</code></td><td>Boolean flag for whether or not the coordinator should execute its various duties of coordinating the cluster. Setting this to true essentially pauses all coordination work while allowing the API to remain up. Duties that are paused include all classes that implement the <code>CoordinatorDuty</code> Interface. Such duties include: Segment balancing, Segment compaction, Emission of metrics controlled by the dynamic coordinator config <code>emitBalancingStats</code>, Submitting kill tasks for unused segments (if enabled), Logging of used segments in the cluster, Marking of newly unused or overshadowed segments, Matching and execution of load/drop rules for used segments, Unloading segments that are no longer marked as used from Historical servers. An example of when an admin may want to pause coordination would be if they are doing deep storage maintenance on HDFS Name Nodes with downtime and don't want the coordinator to be directing Historical Nodes to hit the Name Node with API requests until maintenance is done and the deep store is declared healthy for use again.</td><td>false</td></tr> |
| </tbody> |
| </table> |
| <p>To view the audit history of Coordinator dynamic config issue a GET request to the URL -</p> |
| <pre><code class="hljs">http://<span class="hljs-symbol"><COORDINATOR_IP></span>:<span class="hljs-symbol"><PORT></span>/druid/coordinator/v1/config/<span class="hljs-keyword">history</span>?interval=<span class="hljs-symbol"><interval></span> |
| </code></pre> |
| <p>default value of interval can be specified by setting <code>druid.audit.manager.auditHistoryMillis</code> (1 week if not configured) in Coordinator runtime.properties</p> |
| <p>To view last <n> entries of the audit history of Coordinator dynamic config issue a GET request to the URL -</p> |
| <pre><code class="hljs">http://<span class="hljs-symbol"><COORDINATOR_IP></span>:<span class="hljs-symbol"><PORT></span>/druid/coordinator/v1/config/<span class="hljs-keyword">history</span>?<span class="hljs-built_in">count</span>=<span class="hljs-symbol"><n></span> |
| </code></pre> |
| <h5><a class="anchor" aria-hidden="true" id="lookups-dynamic-configuration"></a><a href="#lookups-dynamic-configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Lookups Dynamic Configuration</h5> |
| <p>These configuration options control the behavior of the Lookup dynamic configuration described in the <a href="/docs/latest/querying/lookups.html">lookups page</a></p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.manager.lookups.hostDeleteTimeout</code></td><td>How long to wait for a <code>DELETE</code> request to a particular process before considering the <code>DELETE</code> a failure</td><td>PT1S</td></tr> |
| <tr><td><code>druid.manager.lookups.hostUpdateTimeout</code></td><td>How long to wait for a <code>POST</code> request to a particular process before considering the <code>POST</code> a failure</td><td>PT10S</td></tr> |
| <tr><td><code>druid.manager.lookups.deleteAllTimeout</code></td><td>How long to wait for all <code>DELETE</code> requests to finish before considering the delete attempt a failure</td><td>PT10S</td></tr> |
| <tr><td><code>druid.manager.lookups.updateAllTimeout</code></td><td>How long to wait for all <code>POST</code> requests to finish before considering the attempt a failure</td><td>PT60S</td></tr> |
| <tr><td><code>druid.manager.lookups.threadPoolSize</code></td><td>How many processes can be managed concurrently (concurrent POST and DELETE requests). Requests this limit will wait in a queue until a slot becomes available.</td><td>10</td></tr> |
| <tr><td><code>druid.manager.lookups.period</code></td><td>How many milliseconds between checks for configuration changes</td><td>30_000</td></tr> |
| </tbody> |
| </table> |
| <h5><a class="anchor" aria-hidden="true" id="compaction-dynamic-configuration"></a><a href="#compaction-dynamic-configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Compaction Dynamic Configuration</h5> |
| <p>Compaction configurations can also be set or updated dynamically using |
| <a href="../operations/api-reference.html#compaction-configuration">Coordinator's API</a> without restarting Coordinators.</p> |
| <p>For details about segment compaction, please check <a href="/docs/latest/operations/segment-optimization.html">Segment Size Optimization</a>.</p> |
| <p>A description of the compaction config is:</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Required</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>dataSource</code></td><td>dataSource name to be compacted.</td><td>yes</td></tr> |
| <tr><td><code>taskPriority</code></td><td><a href="../ingestion/tasks.html#priority">Priority</a> of compaction task.</td><td>no (default = 25)</td></tr> |
| <tr><td><code>inputSegmentSizeBytes</code></td><td>Maximum number of total segment bytes processed per compaction task. Since a time chunk must be processed in its entirety, if the segments for a particular time chunk have a total size in bytes greater than this parameter, compaction will not run for that time chunk. Because each compaction task runs with a single thread, setting this value too far above 1–2GB will result in compaction tasks taking an excessive amount of time.</td><td>no (default = 419430400)</td></tr> |
| <tr><td><code>maxRowsPerSegment</code></td><td>Max number of rows per segment after compaction.</td><td>no</td></tr> |
| <tr><td><code>skipOffsetFromLatest</code></td><td>The offset for searching segments to be compacted. Strongly recommended to set for realtime dataSources.</td><td>no (default = "P1D")</td></tr> |
| <tr><td><code>tuningConfig</code></td><td>Tuning config for compaction tasks. See below <a href="#compaction-tuningconfig">Compaction Task TuningConfig</a>.</td><td>no</td></tr> |
| <tr><td><code>taskContext</code></td><td><a href="../ingestion/tasks.html#context">Task context</a> for compaction tasks.</td><td>no</td></tr> |
| </tbody> |
| </table> |
| <p>An example of compaction config is:</p> |
| <pre><code class="hljs css language-json">{ |
| <span class="hljs-attr">"dataSource"</span>: <span class="hljs-string">"wikiticker"</span> |
| } |
| </code></pre> |
| <p>Note that compaction tasks can fail if their locks are revoked by other tasks of higher priorities. |
| Since realtime tasks have a higher priority than compaction task by default, |
| it can be problematic if there are frequent conflicts between compaction tasks and realtime tasks. |
| If this is the case, the coordinator's automatic compaction might get stuck because of frequent compaction task failures. |
| This kind of problem may happen especially in Kafka/Kinesis indexing systems which allow late data arrival. |
| If you see this problem, it's recommended to set <code>skipOffsetFromLatest</code> to some large enough value to avoid such conflicts between compaction tasks and realtime tasks.</p> |
| <h6><a class="anchor" aria-hidden="true" id="compaction-tuningconfig"></a><a href="#compaction-tuningconfig" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Compaction TuningConfig</h6> |
| <p>Auto compaction supports a subset of the <a href="/docs/latest/ingestion/native-batch.html#tuningconfig">tuningConfig for Parallel task</a>. |
| The below is a list of the supported configurations for auto compaction.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Required</th></tr> |
| </thead> |
| <tbody> |
| <tr><td>type</td><td>The task type, this should always be <code>index_parallel</code>.</td><td>yes</td></tr> |
| <tr><td><code>maxRowsInMemory</code></td><td>Used in determining when intermediate persists to disk should occur. Normally user does not need to set this, but depending on the nature of data, if rows are short in terms of bytes, user may not want to store a million rows in memory and this value should be set.</td><td>no (default = 1000000)</td></tr> |
| <tr><td><code>maxBytesInMemory</code></td><td>Used in determining when intermediate persists to disk should occur. Normally this is computed internally and user does not need to set it. This value represents number of bytes to aggregate in heap memory before persisting. This is based on a rough estimate of memory usage and not actual usage. The maximum heap memory usage for indexing is <code>maxBytesInMemory</code> * (2 + <code>maxPendingPersists</code>)</td><td>no (default = 1/6 of max JVM memory)</td></tr> |
| <tr><td><code>splitHintSpec</code></td><td>Used to give a hint to control the amount of data that each first phase task reads. This hint could be ignored depending on the implementation of the input source. See <a href="/docs/latest/ingestion/native-batch.html#split-hint-spec">Split hint spec</a> for more details.</td><td>no (default = size-based split hint spec)</td></tr> |
| <tr><td><code>partitionsSpec</code></td><td>Defines how to partition data in each time chunk, see <a href="/docs/latest/ingestion/native-batch.html#partitionsspec"><code>PartitionsSpec</code></a></td><td>no (default = <code>dynamic</code>)</td></tr> |
| <tr><td><code>indexSpec</code></td><td>Defines segment storage format options to be used at indexing time, see <a href="/docs/latest/ingestion/index.html#indexspec">IndexSpec</a></td><td>no</td></tr> |
| <tr><td><code>indexSpecForIntermediatePersists</code></td><td>Defines segment storage format options to be used at indexing time for intermediate persisted temporary segments. this can be used to disable dimension/metric compression on intermediate segments to reduce memory required for final merging. however, disabling compression on intermediate segments might increase page cache use while they are used before getting merged into final segment published, see <a href="/docs/latest/ingestion/index.html#indexspec">IndexSpec</a> for possible values.</td><td>no</td></tr> |
| <tr><td><code>maxPendingPersists</code></td><td>Maximum number of persists that can be pending but not started. If this limit would be exceeded by a new intermediate persist, ingestion will block until the currently-running persist finishes. Maximum heap memory usage for indexing scales with <code>maxRowsInMemory</code> * (2 + <code>maxPendingPersists</code>).</td><td>no (default = 0, meaning one persist can be running concurrently with ingestion, and none can be queued up)</td></tr> |
| <tr><td><code>pushTimeout</code></td><td>Milliseconds to wait for pushing segments. It must be >= 0, where 0 means to wait forever.</td><td>no (default = 0)</td></tr> |
| <tr><td><code>segmentWriteOutMediumFactory</code></td><td>Segment write-out medium to use when creating segments. See <a href="/docs/latest/ingestion/native-batch.html#segmentwriteoutmediumfactory">SegmentWriteOutMediumFactory</a>.</td><td>no (default is the value from <code>druid.peon.defaultSegmentWriteOutMediumFactory.type</code> is used)</td></tr> |
| <tr><td><code>maxNumConcurrentSubTasks</code></td><td>Maximum number of worker tasks which can be run in parallel at the same time. The supervisor task would spawn worker tasks up to <code>maxNumConcurrentSubTasks</code> regardless of the current available task slots. If this value is set to 1, the supervisor task processes data ingestion on its own instead of spawning worker tasks. If this value is set to too large, too many worker tasks can be created which might block other ingestion. Check <a href="/docs/latest/ingestion/native-batch.html#capacity-planning">Capacity Planning</a> for more details.</td><td>no (default = 1)</td></tr> |
| <tr><td><code>maxRetry</code></td><td>Maximum number of retries on task failures.</td><td>no (default = 3)</td></tr> |
| <tr><td><code>maxNumSegmentsToMerge</code></td><td>Max limit for the number of segments that a single task can merge at the same time in the second phase. Used only with <code>hashed</code> or <code>single_dim</code> partitionsSpec.</td><td>no (default = 100)</td></tr> |
| <tr><td><code>totalNumMergeTasks</code></td><td>Total number of tasks to merge segments in the merge phase when <code>partitionsSpec</code> is set to <code>hashed</code> or <code>single_dim</code>.</td><td>no (default = 10)</td></tr> |
| <tr><td><code>taskStatusCheckPeriodMs</code></td><td>Polling period in milliseconds to check running task statuses.</td><td>no (default = 1000)</td></tr> |
| <tr><td><code>chatHandlerTimeout</code></td><td>Timeout for reporting the pushed segments in worker tasks.</td><td>no (default = PT10S)</td></tr> |
| <tr><td><code>chatHandlerNumRetries</code></td><td>Retries for reporting the pushed segments in worker tasks.</td><td>no (default = 5)</td></tr> |
| </tbody> |
| </table> |
| <h3><a class="anchor" aria-hidden="true" id="overlord"></a><a href="#overlord" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Overlord</h3> |
| <p>For general Overlord Process information, see <a href="/docs/latest/design/overlord.html">here</a>.</p> |
| <h4><a class="anchor" aria-hidden="true" id="overlord-static-configuration"></a><a href="#overlord-static-configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Overlord Static Configuration</h4> |
| <p>These Overlord static configurations can be defined in the <code>overlord/runtime.properties</code> file.</p> |
| <h5><a class="anchor" aria-hidden="true" id="overlord-process-configs"></a><a href="#overlord-process-configs" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Overlord Process Configs</h5> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.host</code></td><td>The host for the current process. This is used to advertise the current processes location as reachable from another process and should generally be specified such that <code>http://${druid.host}/</code> could actually talk to this process</td><td>InetAddress.getLocalHost().getCanonicalHostName()</td></tr> |
| <tr><td><code>druid.bindOnHost</code></td><td>Indicating whether the process's internal jetty server bind on <code>druid.host</code>. Default is false, which means binding to all interfaces.</td><td>false</td></tr> |
| <tr><td><code>druid.plaintextPort</code></td><td>This is the port to actually listen on; unless port mapping is used, this will be the same port as is on <code>druid.host</code></td><td>8090</td></tr> |
| <tr><td><code>druid.tlsPort</code></td><td>TLS port for HTTPS connector, if <a href="/docs/latest/operations/tls-support.html">druid.enableTlsPort</a> is set then this config will be used. If <code>druid.host</code> contains port then that port will be ignored. This should be a non-negative Integer.</td><td>8290</td></tr> |
| <tr><td><code>druid.service</code></td><td>The name of the service. This is used as a dimension when emitting metrics and alerts to differentiate between the various services</td><td>druid/overlord</td></tr> |
| </tbody> |
| </table> |
| <h5><a class="anchor" aria-hidden="true" id="overlord-operations"></a><a href="#overlord-operations" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Overlord Operations</h5> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.indexer.runner.type</code></td><td>Choices "local" or "remote". Indicates whether tasks should be run locally or in a distributed environment. Experimental task runner "httpRemote" is also available which is same as "remote" but uses HTTP to interact with Middle Managers instead of Zookeeper.</td><td>local</td></tr> |
| <tr><td><code>druid.indexer.storage.type</code></td><td>Choices are "local" or "metadata". Indicates whether incoming tasks should be stored locally (in heap) or in metadata storage. Storing incoming tasks in metadata storage allows for tasks to be resumed if the Overlord should fail.</td><td>local</td></tr> |
| <tr><td><code>druid.indexer.storage.recentlyFinishedThreshold</code></td><td>A duration of time to store task results.</td><td>PT24H</td></tr> |
| <tr><td><code>druid.indexer.tasklock.forceTimeChunkLock</code></td><td><em><strong>Setting this to false is still experimental</strong></em><br/> If set, all tasks are enforced to use time chunk lock. If not set, each task automatically chooses a lock type to use. This configuration can be overwritten by setting <code>forceTimeChunkLock</code> in the <a href="/docs/latest/ingestion/tasks.html#context">task context</a>. See <a href="/docs/latest/ingestion/tasks.html#context">Task Locking & Priority</a> for more details about locking in tasks.</td><td>true</td></tr> |
| <tr><td><code>druid.indexer.queue.maxSize</code></td><td>Maximum number of active tasks at one time.</td><td>Integer.MAX_VALUE</td></tr> |
| <tr><td><code>druid.indexer.queue.startDelay</code></td><td>Sleep this long before starting Overlord queue management. This can be useful to give a cluster time to re-orient itself after e.g. a widespread network issue.</td><td>PT1M</td></tr> |
| <tr><td><code>druid.indexer.queue.restartDelay</code></td><td>Sleep this long when Overlord queue management throws an exception before trying again.</td><td>PT30S</td></tr> |
| <tr><td><code>druid.indexer.queue.storageSyncRate</code></td><td>Sync Overlord state this often with an underlying task persistence mechanism.</td><td>PT1M</td></tr> |
| </tbody> |
| </table> |
| <p>The following configs only apply if the Overlord is running in remote mode. For a description of local vs. remote mode, please see (../design/overlord.html).</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.indexer.runner.taskAssignmentTimeout</code></td><td>How long to wait after a task as been assigned to a MiddleManager before throwing an error.</td><td>PT5M</td></tr> |
| <tr><td><code>druid.indexer.runner.minWorkerVersion</code></td><td>The minimum MiddleManager version to send tasks to.</td><td>"0"</td></tr> |
| <tr><td><code>druid.indexer.runner.compressZnodes</code></td><td>Indicates whether or not the Overlord should expect MiddleManagers to compress Znodes.</td><td>true</td></tr> |
| <tr><td><code>druid.indexer.runner.maxZnodeBytes</code></td><td>The maximum size Znode in bytes that can be created in Zookeeper. Human-readable format is supported, see <a href="/docs/latest/configuration/human-readable-byte.html">here</a>.</td><td>512 KiB</td></tr> |
| <tr><td><code>druid.indexer.runner.taskCleanupTimeout</code></td><td>How long to wait before failing a task after a MiddleManager is disconnected from Zookeeper.</td><td>PT15M</td></tr> |
| <tr><td><code>druid.indexer.runner.taskShutdownLinkTimeout</code></td><td>How long to wait on a shutdown request to a MiddleManager before timing out</td><td>PT1M</td></tr> |
| <tr><td><code>druid.indexer.runner.pendingTasksRunnerNumThreads</code></td><td>Number of threads to allocate pending-tasks to workers, must be at least 1.</td><td>1</td></tr> |
| <tr><td><code>druid.indexer.runner.maxRetriesBeforeBlacklist</code></td><td>Number of consecutive times the MiddleManager can fail tasks, before the worker is blacklisted, must be at least 1</td><td>5</td></tr> |
| <tr><td><code>druid.indexer.runner.workerBlackListBackoffTime</code></td><td>How long to wait before a task is whitelisted again. This value should be greater that the value set for taskBlackListCleanupPeriod.</td><td>PT15M</td></tr> |
| <tr><td><code>druid.indexer.runner.workerBlackListCleanupPeriod</code></td><td>A duration after which the cleanup thread will startup to clean blacklisted workers.</td><td>PT5M</td></tr> |
| <tr><td><code>druid.indexer.runner.maxPercentageBlacklistWorkers</code></td><td>The maximum percentage of workers to blacklist, this must be between 0 and 100.</td><td>20</td></tr> |
| </tbody> |
| </table> |
| <p>There are additional configs for autoscaling (if it is enabled):</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.indexer.autoscale.strategy</code></td><td>Choices are "noop", "ec2" or "gce". Sets the strategy to run when autoscaling is required.</td><td>noop</td></tr> |
| <tr><td><code>druid.indexer.autoscale.doAutoscale</code></td><td>If set to "true" autoscaling will be enabled.</td><td>false</td></tr> |
| <tr><td><code>druid.indexer.autoscale.provisionPeriod</code></td><td>How often to check whether or not new MiddleManagers should be added.</td><td>PT1M</td></tr> |
| <tr><td><code>druid.indexer.autoscale.terminatePeriod</code></td><td>How often to check when MiddleManagers should be removed.</td><td>PT5M</td></tr> |
| <tr><td><code>druid.indexer.autoscale.originTime</code></td><td>The starting reference timestamp that the terminate period increments upon.</td><td>2012-01-01T00:55:00.000Z</td></tr> |
| <tr><td><code>druid.indexer.autoscale.workerIdleTimeout</code></td><td>How long can a worker be idle (not a run task) before it can be considered for termination.</td><td>PT90M</td></tr> |
| <tr><td><code>druid.indexer.autoscale.maxScalingDuration</code></td><td>How long the Overlord will wait around for a MiddleManager to show up before giving up.</td><td>PT15M</td></tr> |
| <tr><td><code>druid.indexer.autoscale.numEventsToTrack</code></td><td>The number of autoscaling related events (node creation and termination) to track.</td><td>10</td></tr> |
| <tr><td><code>druid.indexer.autoscale.pendingTaskTimeout</code></td><td>How long a task can be in "pending" state before the Overlord tries to scale up.</td><td>PT30S</td></tr> |
| <tr><td><code>druid.indexer.autoscale.workerVersion</code></td><td>If set, will only create nodes of set version during autoscaling. Overrides dynamic configuration.</td><td>null</td></tr> |
| <tr><td><code>druid.indexer.autoscale.workerPort</code></td><td>The port that MiddleManagers will run on.</td><td>8080</td></tr> |
| </tbody> |
| </table> |
| <h5><a class="anchor" aria-hidden="true" id="supervisors"></a><a href="#supervisors" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Supervisors</h5> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.supervisor.healthinessThreshold</code></td><td>The number of successful runs before an unhealthy supervisor is again considered healthy.</td><td>3</td></tr> |
| <tr><td><code>druid.supervisor.unhealthinessThreshold</code></td><td>The number of failed runs before the supervisor is considered unhealthy.</td><td>3</td></tr> |
| <tr><td><code>druid.supervisor.taskHealthinessThreshold</code></td><td>The number of consecutive task successes before an unhealthy supervisor is again considered healthy.</td><td>3</td></tr> |
| <tr><td><code>druid.supervisor.taskUnhealthinessThreshold</code></td><td>The number of consecutive task failures before the supervisor is considered unhealthy.</td><td>3</td></tr> |
| <tr><td><code>druid.supervisor.storeStackTrace</code></td><td>Whether full stack traces of supervisor exceptions should be stored and returned by the supervisor <code>/status</code> endpoint.</td><td>false</td></tr> |
| <tr><td><code>druid.supervisor.maxStoredExceptionEvents</code></td><td>The maximum number of exception events that can be returned through the supervisor <code>/status</code> endpoint.</td><td><code>max(healthinessThreshold, unhealthinessThreshold)</code></td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="overlord-dynamic-configuration"></a><a href="#overlord-dynamic-configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Overlord Dynamic Configuration</h4> |
| <p>The Overlord can dynamically change worker behavior.</p> |
| <p>The JSON object can be submitted to the Overlord via a POST request at:</p> |
| <pre><code class="hljs">http:<span class="hljs-regexp">//</span><OVERLORD_IP>:<port><span class="hljs-regexp">/druid/i</span>ndexer<span class="hljs-regexp">/v1/</span>worker |
| </code></pre> |
| <p>Optional Header Parameters for auditing the config change can also be specified.</p> |
| <table> |
| <thead> |
| <tr><th>Header Param Name</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>X-Druid-Author</code></td><td>author making the config change</td><td>""</td></tr> |
| <tr><td><code>X-Druid-Comment</code></td><td>comment describing the change being done</td><td>""</td></tr> |
| </tbody> |
| </table> |
| <p>A sample worker config spec is shown below:</p> |
| <pre><code class="hljs css language-json">{ |
| <span class="hljs-attr">"selectStrategy"</span>: { |
| <span class="hljs-attr">"type"</span>: <span class="hljs-string">"fillCapacity"</span>, |
| <span class="hljs-attr">"affinityConfig"</span>: { |
| <span class="hljs-attr">"affinity"</span>: { |
| <span class="hljs-attr">"datasource1"</span>: [<span class="hljs-string">"host1:port"</span>, <span class="hljs-string">"host2:port"</span>], |
| <span class="hljs-attr">"datasource2"</span>: [<span class="hljs-string">"host3:port"</span>] |
| } |
| } |
| }, |
| <span class="hljs-attr">"autoScaler"</span>: { |
| <span class="hljs-attr">"type"</span>: <span class="hljs-string">"ec2"</span>, |
| <span class="hljs-attr">"minNumWorkers"</span>: <span class="hljs-number">2</span>, |
| <span class="hljs-attr">"maxNumWorkers"</span>: <span class="hljs-number">12</span>, |
| <span class="hljs-attr">"envConfig"</span>: { |
| <span class="hljs-attr">"availabilityZone"</span>: <span class="hljs-string">"us-east-1a"</span>, |
| <span class="hljs-attr">"nodeData"</span>: { |
| <span class="hljs-attr">"amiId"</span>: <span class="hljs-string">"${AMI}"</span>, |
| <span class="hljs-attr">"instanceType"</span>: <span class="hljs-string">"c3.8xlarge"</span>, |
| <span class="hljs-attr">"minInstances"</span>: <span class="hljs-number">1</span>, |
| <span class="hljs-attr">"maxInstances"</span>: <span class="hljs-number">1</span>, |
| <span class="hljs-attr">"securityGroupIds"</span>: [<span class="hljs-string">"${IDs}"</span>], |
| <span class="hljs-attr">"keyName"</span>: <span class="hljs-string">"${KEY_NAME}"</span> |
| }, |
| <span class="hljs-attr">"userData"</span>: { |
| <span class="hljs-attr">"impl"</span>: <span class="hljs-string">"string"</span>, |
| <span class="hljs-attr">"data"</span>: <span class="hljs-string">"${SCRIPT_COMMAND}"</span>, |
| <span class="hljs-attr">"versionReplacementString"</span>: <span class="hljs-string">":VERSION:"</span>, |
| <span class="hljs-attr">"version"</span>: <span class="hljs-literal">null</span> |
| } |
| } |
| } |
| } |
| </code></pre> |
| <p>Issuing a GET request at the same URL will return the current worker config spec that is currently in place. The worker config spec list above is just a sample for EC2 and it is possible to extend the code base for other deployment environments. A description of the worker config spec is shown below.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>selectStrategy</code></td><td>How to assign tasks to MiddleManagers. Choices are <code>fillCapacity</code>, <code>equalDistribution</code>, and <code>javascript</code>.</td><td>equalDistribution</td></tr> |
| <tr><td><code>autoScaler</code></td><td>Only used if autoscaling is enabled. See below.</td><td>null</td></tr> |
| </tbody> |
| </table> |
| <p>To view the audit history of worker config issue a GET request to the URL -</p> |
| <pre><code class="hljs">http://<span class="hljs-symbol"><OVERLORD_IP></span>:<span class="hljs-symbol"><port></span>/druid/indexer/v1/worker/<span class="hljs-keyword">history</span>?interval=<span class="hljs-symbol"><interval></span> |
| </code></pre> |
| <p>default value of interval can be specified by setting <code>druid.audit.manager.auditHistoryMillis</code> (1 week if not configured) in Overlord runtime.properties.</p> |
| <p>To view last <n> entries of the audit history of worker config issue a GET request to the URL -</p> |
| <pre><code class="hljs">http://<span class="hljs-symbol"><OVERLORD_IP></span>:<span class="hljs-symbol"><port></span>/druid/indexer/v1/worker/<span class="hljs-keyword">history</span>?<span class="hljs-built_in">count</span>=<span class="hljs-symbol"><n></span> |
| </code></pre> |
| <h5><a class="anchor" aria-hidden="true" id="worker-select-strategy"></a><a href="#worker-select-strategy" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Worker Select Strategy</h5> |
| <p>Worker select strategies control how Druid assigns tasks to MiddleManagers.</p> |
| <h6><a class="anchor" aria-hidden="true" id="equal-distribution"></a><a href="#equal-distribution" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Equal Distribution</h6> |
| <p>Tasks are assigned to the MiddleManager with the most free slots at the time the task begins running. This is useful if |
| you want work evenly distributed across your MiddleManagers.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>type</code></td><td><code>equalDistribution</code>.</td><td>required; must be <code>equalDistribution</code></td></tr> |
| <tr><td><code>affinityConfig</code></td><td><a href="#affinity">Affinity config</a> object</td><td>null (no affinity)</td></tr> |
| </tbody> |
| </table> |
| <h6><a class="anchor" aria-hidden="true" id="equal-distribution-with-category-spec"></a><a href="#equal-distribution-with-category-spec" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Equal Distribution With Category Spec</h6> |
| <p>This strategy is a variant of <code>Equal Distribution</code>, which support <code>workerCategorySpec</code> field rather than <code>affinityConfig</code>. By specifying <code>workerCategorySpec</code>, you can assign tasks to run on different categories of MiddleManagers based on the tasks' <strong>taskType</strong> and <strong>dataSource name</strong>. This strategy can't work with <code>AutoScaler</code> since the behavior is undefined.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>type</code></td><td><code>equalDistributionWithCategorySpec</code>.</td><td>required; must be <code>equalDistributionWithCategorySpec</code></td></tr> |
| <tr><td><code>workerCategorySpec</code></td><td><a href="#workercategoryspec">Worker Category Spec</a> object</td><td>null (no worker category spec)</td></tr> |
| </tbody> |
| </table> |
| <p>Example: specify tasks default to run on <strong>c1</strong> whose task |
| type is "index_kafka", while dataSource "ds1" run on <strong>c2</strong>.</p> |
| <pre><code class="hljs css language-json">{ |
| <span class="hljs-attr">"selectStrategy"</span>: { |
| <span class="hljs-attr">"type"</span>: <span class="hljs-string">"equalDistributionWithCategorySpec"</span>, |
| <span class="hljs-attr">"workerCategorySpec"</span>: { |
| <span class="hljs-attr">"strong"</span>: <span class="hljs-literal">false</span>, |
| <span class="hljs-attr">"categoryMap"</span>: { |
| <span class="hljs-attr">"index_kafka"</span>: { |
| <span class="hljs-attr">"defaultCategory"</span>: <span class="hljs-string">"c1"</span>, |
| <span class="hljs-attr">"categoryAffinity"</span>: { |
| <span class="hljs-attr">"ds1"</span>: <span class="hljs-string">"c2"</span> |
| } |
| } |
| } |
| } |
| } |
| } |
| </code></pre> |
| <h6><a class="anchor" aria-hidden="true" id="fill-capacity"></a><a href="#fill-capacity" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Fill Capacity</h6> |
| <p>Tasks are assigned to the worker with the most currently-running tasks at the time the task begins running. This is |
| useful in situations where you are elastically auto-scaling MiddleManagers, since it will tend to pack some full and |
| leave others empty. The empty ones can be safely terminated.</p> |
| <p>Note that if <code>druid.indexer.runner.pendingTasksRunnerNumThreads</code> is set to <em>N</em> > 1, then this strategy will fill <em>N</em> |
| MiddleManagers up to capacity simultaneously, rather than a single MiddleManager.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>type</code></td><td><code>fillCapacity</code>.</td><td>required; must be <code>fillCapacity</code></td></tr> |
| <tr><td><code>affinityConfig</code></td><td><a href="#affinity">Affinity config</a> object</td><td>null (no affinity)</td></tr> |
| </tbody> |
| </table> |
| <h6><a class="anchor" aria-hidden="true" id="fill-capacity-with-category-spec"></a><a href="#fill-capacity-with-category-spec" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Fill Capacity With Category Spec</h6> |
| <p>This strategy is a variant of <code>Fill Capacity</code>, which support <code>workerCategorySpec</code> field rather than <code>affinityConfig</code>. The usage is the same with <em>equalDistributionWithCategorySpec</em> strategy. This strategy can't work with <code>AutoScaler</code> since the behavior is undefined.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>type</code></td><td><code>fillCapacityWithCategorySpec</code>.</td><td>required; must be <code>fillCapacityWithCategorySpec</code></td></tr> |
| <tr><td><code>workerCategorySpec</code></td><td><a href="#workercategoryspec">Worker Category Spec</a> object</td><td>null (no worker category spec)</td></tr> |
| </tbody> |
| </table> |
| <blockquote> |
| <p>Before using the <em>equalDistributionWithCategorySpec</em> and <em>fillCapacityWithCategorySpec</em> strategies, you must upgrade overlord and all MiddleManagers to the version that support this feature.</p> |
| </blockquote> |
| <p><a name="javascript-worker-select-strategy"></a></p> |
| <h6><a class="anchor" aria-hidden="true" id="javascript-1"></a><a href="#javascript-1" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>JavaScript</h6> |
| <p>Allows defining arbitrary logic for selecting workers to run task using a JavaScript function. |
| The function is passed remoteTaskRunnerConfig, map of workerId to available workers and task to be executed and returns the workerId on which the task should be run or null if the task cannot be run. |
| It can be used for rapid development of missing features where the worker selection logic is to be changed or tuned often. |
| If the selection logic is quite complex and cannot be easily tested in JavaScript environment, |
| its better to write a druid extension module with extending current worker selection strategies written in java.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>type</code></td><td><code>javascript</code>.</td><td>required; must be <code>javascript</code></td></tr> |
| <tr><td><code>function</code></td><td>String representing JavaScript function</td><td></td></tr> |
| </tbody> |
| </table> |
| <p>Example: a function that sends batch_index_task to workers 10.0.0.1 and 10.0.0.2 and all other tasks to other available workers.</p> |
| <pre><code class="hljs">{ |
| "type":"javascript", |
| "function":"function (config, zkWorkers, task) {<span class="hljs-symbol">\n</span>var batch_workers = new java.util.ArrayList();<span class="hljs-symbol">\n</span>batch_workers.add(<span class="hljs-symbol">\"</span>middleManager1_hostname:8091<span class="hljs-symbol">\"</span>);<span class="hljs-symbol">\n</span>batch_workers.add(<span class="hljs-symbol">\"</span>middleManager2_hostname:8091<span class="hljs-symbol">\"</span>);<span class="hljs-symbol">\n</span>workers = zkWorkers.keySet().toArray();<span class="hljs-symbol">\n</span>var sortedWorkers = new Array()<span class="hljs-symbol">\n</span>;for(var i = 0; i < workers.length; i++){<span class="hljs-symbol">\n</span> sortedWorkers[i] = workers[i];<span class="hljs-symbol">\n</span>}<span class="hljs-symbol">\n</span>Array.prototype.sort.call(sortedWorkers,function(a, b){return zkWorkers.get(b).getCurrCapacityUsed() - zkWorkers.get(a).getCurrCapacityUsed();});<span class="hljs-symbol">\n</span>var minWorkerVer = config.getMinWorkerVersion();<span class="hljs-symbol">\n</span>for (var i = 0; i < sortedWorkers.length; i++) {<span class="hljs-symbol">\n</span> var worker = sortedWorkers[i];<span class="hljs-symbol">\n</span> var zkWorker = zkWorkers.get(worker);<span class="hljs-symbol">\n</span> if(zkWorker.canRunTask(task) && zkWorker.isValidVersion(minWorkerVer)){<span class="hljs-symbol">\n</span> if(task.getType() == 'index_hadoop' && batch_workers.contains(worker)){<span class="hljs-symbol">\n</span> return worker;<span class="hljs-symbol">\n</span> } else {<span class="hljs-symbol">\n</span> if(task.getType() != 'index_hadoop' && !batch_workers.contains(worker)){<span class="hljs-symbol">\n</span> return worker;<span class="hljs-symbol">\n</span> }<span class="hljs-symbol">\n</span> }<span class="hljs-symbol">\n</span> }<span class="hljs-symbol">\n</span>}<span class="hljs-symbol">\n</span>return null;<span class="hljs-symbol">\n</span>}" |
| } |
| </code></pre> |
| <blockquote> |
| <p>JavaScript-based functionality is disabled by default. Please refer to the Druid <a href="/docs/latest/development/javascript.html">JavaScript programming guide</a> for guidelines about using Druid's JavaScript functionality, including instructions on how to enable it.</p> |
| </blockquote> |
| <h6><a class="anchor" aria-hidden="true" id="affinity"></a><a href="#affinity" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Affinity</h6> |
| <p>Affinity configs can be provided to the <em>equalDistribution</em> and <em>fillCapacity</em> strategies using the "affinityConfig" |
| field. If not provided, the default is to not use affinity at all.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>affinity</code></td><td>JSON object mapping a datasource String name to a list of indexing service MiddleManager host:port String values. Druid doesn't perform DNS resolution, so the 'host' value must match what is configured on the MiddleManager and what the MiddleManager announces itself as (examine the Overlord logs to see what your MiddleManager announces itself as).</td><td>{}</td></tr> |
| <tr><td><code>strong</code></td><td>With weak affinity (the default), tasks for a dataSource may be assigned to other MiddleManagers if their affinity-mapped MiddleManagers are not able to run all pending tasks in the queue for that dataSource. With strong affinity, tasks for a dataSource will only ever be assigned to their affinity-mapped MiddleManagers, and will wait in the pending queue if necessary.</td><td>false</td></tr> |
| </tbody> |
| </table> |
| <h6><a class="anchor" aria-hidden="true" id="workercategoryspec"></a><a href="#workercategoryspec" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>WorkerCategorySpec</h6> |
| <p>WorkerCategorySpec can be provided to the <em>equalDistributionWithCategorySpec</em> and <em>fillCapacityWithCategorySpec</em> strategies using the "workerCategorySpec" |
| field. If not provided, the default is to not use it at all.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>categoryMap</code></td><td>A JSON map object mapping a task type String name to a <a href="#categoryconfig">CategoryConfig</a> object, by which you can specify category config for different task type.</td><td>{}</td></tr> |
| <tr><td><code>strong</code></td><td>With weak workerCategorySpec (the default), tasks for a dataSource may be assigned to other MiddleManagers if the MiddleManagers specified in <code>categoryMap</code> are not able to run all pending tasks in the queue for that dataSource. With strong workerCategorySpec, tasks for a dataSource will only ever be assigned to their specified MiddleManagers, and will wait in the pending queue if necessary.</td><td>false</td></tr> |
| </tbody> |
| </table> |
| <h6><a class="anchor" aria-hidden="true" id="categoryconfig"></a><a href="#categoryconfig" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>CategoryConfig</h6> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>defaultCategory</code></td><td>Specify default category for a task type.</td><td>null</td></tr> |
| <tr><td><code>categoryAffinity</code></td><td>A JSON map object mapping a datasource String name to a category String name of the MiddleManager. If category isn't specified for a datasource, then using the <code>defaultCategory</code>. If no specified category and the <code>defaultCategory</code> is also null, then tasks can run on any available MiddleManagers.</td><td>null</td></tr> |
| </tbody> |
| </table> |
| <h5><a class="anchor" aria-hidden="true" id="autoscaler"></a><a href="#autoscaler" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Autoscaler</h5> |
| <p>Amazon's EC2 together with Google's GCE are currently the only supported autoscalers.</p> |
| <p>EC2's autoscaler properties are:</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>minNumWorkers</code></td><td>The minimum number of workers that can be in the cluster at any given time.</td><td>0</td></tr> |
| <tr><td><code>maxNumWorkers</code></td><td>The maximum number of workers that can be in the cluster at any given time.</td><td>0</td></tr> |
| <tr><td><code>availabilityZone</code></td><td>What availability zone to run in.</td><td>none</td></tr> |
| <tr><td><code>nodeData</code></td><td>A JSON object that describes how to launch new nodes.</td><td>none; required</td></tr> |
| <tr><td><code>userData</code></td><td>A JSON object that describes how to configure new nodes. If you have set druid.indexer.autoscale.workerVersion, this must have a versionReplacementString. Otherwise, a versionReplacementString is not necessary.</td><td>none; optional</td></tr> |
| </tbody> |
| </table> |
| <p>For GCE's properties, please refer to the <a href="/docs/latest/development/extensions-contrib/gce-extensions.html">gce-extensions</a>.</p> |
| <h2><a class="anchor" aria-hidden="true" id="data-server"></a><a href="#data-server" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Data Server</h2> |
| <p>This section contains the configuration options for the processes that reside on Data servers (MiddleManagers/Peons and Historicals) in the suggested <a href="../design/processes.html#server-types">three-server configuration</a>.</p> |
| <p>Configuration options for the experimental <a href="/docs/latest/design/indexer.html">Indexer process</a> are also provided here.</p> |
| <h3><a class="anchor" aria-hidden="true" id="middlemanager-and-peons"></a><a href="#middlemanager-and-peons" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>MiddleManager and Peons</h3> |
| <p>These MiddleManager and Peon configurations can be defined in the <code>middleManager/runtime.properties</code> file.</p> |
| <h4><a class="anchor" aria-hidden="true" id="middlemanager-process-config"></a><a href="#middlemanager-process-config" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>MiddleManager Process Config</h4> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.host</code></td><td>The host for the current process. This is used to advertise the current processes location as reachable from another process and should generally be specified such that <code>http://${druid.host}/</code> could actually talk to this process</td><td>InetAddress.getLocalHost().getCanonicalHostName()</td></tr> |
| <tr><td><code>druid.bindOnHost</code></td><td>Indicating whether the process's internal jetty server bind on <code>druid.host</code>. Default is false, which means binding to all interfaces.</td><td>false</td></tr> |
| <tr><td><code>druid.plaintextPort</code></td><td>This is the port to actually listen on; unless port mapping is used, this will be the same port as is on <code>druid.host</code></td><td>8091</td></tr> |
| <tr><td><code>druid.tlsPort</code></td><td>TLS port for HTTPS connector, if <a href="/docs/latest/operations/tls-support.html">druid.enableTlsPort</a> is set then this config will be used. If <code>druid.host</code> contains port then that port will be ignored. This should be a non-negative Integer.</td><td>8291</td></tr> |
| <tr><td><code>druid.service</code></td><td>The name of the service. This is used as a dimension when emitting metrics and alerts to differentiate between the various services</td><td>druid/middlemanager</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="middlemanager-configuration"></a><a href="#middlemanager-configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>MiddleManager Configuration</h4> |
| <p>Middle managers pass their configurations down to their child peons. The MiddleManager requires the following configs:</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.indexer.runner.allowedPrefixes</code></td><td>Whitelist of prefixes for configs that can be passed down to child peons.</td><td>"com.metamx", "druid", "org.apache.druid", "user.timezone", "file.encoding", "java.io.tmpdir", "hadoop"</td></tr> |
| <tr><td><code>druid.indexer.runner.compressZnodes</code></td><td>Indicates whether or not the MiddleManagers should compress Znodes.</td><td>true</td></tr> |
| <tr><td><code>druid.indexer.runner.classpath</code></td><td>Java classpath for the peon.</td><td>System.getProperty("java.class.path")</td></tr> |
| <tr><td><code>druid.indexer.runner.javaCommand</code></td><td>Command required to execute java.</td><td>java</td></tr> |
| <tr><td><code>druid.indexer.runner.javaOpts</code></td><td><em>DEPRECATED</em> A string of -X Java options to pass to the peon's JVM. Quotable parameters or parameters with spaces are encouraged to use javaOptsArray</td><td>""</td></tr> |
| <tr><td><code>druid.indexer.runner.javaOptsArray</code></td><td>A JSON array of strings to be passed in as options to the peon's JVM. This is additive to javaOpts and is recommended for properly handling arguments which contain quotes or spaces like <code>["-XX:OnOutOfMemoryError=kill -9 %p"]</code></td><td><code>[]</code></td></tr> |
| <tr><td><code>druid.indexer.runner.maxZnodeBytes</code></td><td>The maximum size Znode in bytes that can be created in Zookeeper.</td><td>524288</td></tr> |
| <tr><td><code>druid.indexer.runner.startPort</code></td><td>Starting port used for peon processes, should be greater than 1023 and less than 65536.</td><td>8100</td></tr> |
| <tr><td><code>druid.indexer.runner.endPort</code></td><td>Ending port used for peon processes, should be greater than or equal to <code>druid.indexer.runner.startPort</code> and less than 65536.</td><td>65535</td></tr> |
| <tr><td><code>druid.indexer.runner.ports</code></td><td>A JSON array of integers to specify ports that used for peon processes. If provided and non-empty, ports for peon processes will be chosen from these ports. And <code>druid.indexer.runner.startPort/druid.indexer.runner.endPort</code> will be completely ignored.</td><td><code>[]</code></td></tr> |
| <tr><td><code>druid.worker.ip</code></td><td>The IP of the worker.</td><td>localhost</td></tr> |
| <tr><td><code>druid.worker.version</code></td><td>Version identifier for the MiddleManager.</td><td>0</td></tr> |
| <tr><td><code>druid.worker.capacity</code></td><td>Maximum number of tasks the MiddleManager can accept.</td><td>Number of CPUs on the machine - 1</td></tr> |
| <tr><td><code>druid.worker.category</code></td><td>A string to name the category that the MiddleManager node belongs to.</td><td><code>__default_worker_category</code></td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="peon-processing"></a><a href="#peon-processing" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Peon Processing</h4> |
| <p>Processing properties set on the Middlemanager will be passed through to Peons.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.processing.buffer.sizeBytes</code></td><td>This specifies a buffer size for the storage of intermediate results. The computation engine in both the Historical and Realtime processes will use a scratch buffer of this size to do all of their intermediate computations off-heap. Larger values allow for more aggregations in a single pass over the data while smaller values can require more passes depending on the query that is being executed. Human-readable format is supported, see <a href="/docs/latest/configuration/human-readable-byte.html">here</a>.</td><td>auto (max 1 GiB)</td></tr> |
| <tr><td><code>druid.processing.buffer.poolCacheMaxCount</code></td><td>processing buffer pool caches the buffers for later use, this is the maximum count cache will grow to. note that pool can create more buffers than it can cache if necessary.</td><td>Integer.MAX_VALUE</td></tr> |
| <tr><td><code>druid.processing.formatString</code></td><td>Realtime and Historical processes use this format string to name their processing threads.</td><td>processing-%s</td></tr> |
| <tr><td><code>druid.processing.numMergeBuffers</code></td><td>The number of direct memory buffers available for merging query results. The buffers are sized by <code>druid.processing.buffer.sizeBytes</code>. This property is effectively a concurrency limit for queries that require merging buffers. If you are using any queries that require merge buffers (currently, just groupBy v2) then you should have at least two of these.</td><td><code>max(2, druid.processing.numThreads / 4)</code></td></tr> |
| <tr><td><code>druid.processing.numThreads</code></td><td>The number of processing threads to have available for parallel processing of segments. Our rule of thumb is <code>num_cores - 1</code>, which means that even under heavy load there will still be one core available to do background tasks like talking with ZooKeeper and pulling down segments. If only one core is available, this property defaults to the value <code>1</code>.</td><td>Number of cores - 1 (or 1)</td></tr> |
| <tr><td><code>druid.processing.columnCache.sizeBytes</code></td><td>Maximum size in bytes for the dimension value lookup cache. Any value greater than <code>0</code> enables the cache. It is currently disabled by default. Enabling the lookup cache can significantly improve the performance of aggregators operating on dimension values, such as the JavaScript aggregator, or cardinality aggregator, but can slow things down if the cache hit rate is low (i.e. dimensions with few repeating values). Enabling it may also require additional garbage collection tuning to avoid long GC pauses.</td><td><code>0</code> (disabled)</td></tr> |
| <tr><td><code>druid.processing.fifo</code></td><td>If the processing queue should treat tasks of equal priority in a FIFO manner</td><td><code>false</code></td></tr> |
| <tr><td><code>druid.processing.tmpDir</code></td><td>Path where temporary files created while processing a query should be stored. If specified, this configuration takes priority over the default <code>java.io.tmpdir</code> path.</td><td>path represented by <code>java.io.tmpdir</code></td></tr> |
| </tbody> |
| </table> |
| <p>The amount of direct memory needed by Druid is at least |
| <code>druid.processing.buffer.sizeBytes * (druid.processing.numMergeBuffers + druid.processing.numThreads + 1)</code>. You can |
| ensure at least this amount of direct memory is available by providing <code>-XX:MaxDirectMemorySize=<VALUE></code> in |
| <code>druid.indexer.runner.javaOptsArray</code> as documented above.</p> |
| <h4><a class="anchor" aria-hidden="true" id="peon-query-configuration"></a><a href="#peon-query-configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Peon query configuration</h4> |
| <p>See <a href="#general-query-configuration">general query configuration</a>.</p> |
| <h4><a class="anchor" aria-hidden="true" id="peon-caching"></a><a href="#peon-caching" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Peon Caching</h4> |
| <p>You can optionally configure caching to be enabled on the peons by setting caching configs here.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Possible Values</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.realtime.cache.useCache</code></td><td>true, false</td><td>Enable the cache on the realtime.</td><td>false</td></tr> |
| <tr><td><code>druid.realtime.cache.populateCache</code></td><td>true, false</td><td>Populate the cache on the realtime.</td><td>false</td></tr> |
| <tr><td><code>druid.realtime.cache.unCacheable</code></td><td>All druid query types</td><td>All query types to not cache.</td><td><code>[]</code></td></tr> |
| <tr><td><code>druid.realtime.cache.maxEntrySize</code></td><td>positive integer</td><td>Maximum cache entry size in bytes.</td><td>1_000_000</td></tr> |
| </tbody> |
| </table> |
| <p>See <a href="#cache-configuration">cache configuration</a> for how to configure cache settings.</p> |
| <h4><a class="anchor" aria-hidden="true" id="additional-peon-configuration"></a><a href="#additional-peon-configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Additional Peon Configuration</h4> |
| <p>Although peons inherit the configurations of their parent MiddleManagers, explicit child peon configs in MiddleManager can be set by prefixing them with:</p> |
| <pre><code class="hljs">druid.indexer.fork.<span class="hljs-keyword">property</span><span class="hljs-title"> |
| </span></code></pre> |
| <p>Additional peon configs include:</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.peon.mode</code></td><td>Choices are "local" and "remote". Setting this to local means you intend to run the peon as a standalone process (Not recommended).</td><td>remote</td></tr> |
| <tr><td><code>druid.indexer.task.baseDir</code></td><td>Base temporary working directory.</td><td><code>System.getProperty("java.io.tmpdir")</code></td></tr> |
| <tr><td><code>druid.indexer.task.baseTaskDir</code></td><td>Base temporary working directory for tasks.</td><td><code>${druid.indexer.task.baseDir}/persistent/task</code></td></tr> |
| <tr><td><code>druid.indexer.task.defaultHadoopCoordinates</code></td><td>Hadoop version to use with HadoopIndexTasks that do not request a particular version.</td><td>org.apache.hadoop:hadoop-client:2.8.5</td></tr> |
| <tr><td><code>druid.indexer.task.defaultRowFlushBoundary</code></td><td>Highest row count before persisting to disk. Used for indexing generating tasks.</td><td>75000</td></tr> |
| <tr><td><code>druid.indexer.task.directoryLockTimeout</code></td><td>Wait this long for zombie peons to exit before giving up on their replacements.</td><td>PT10M</td></tr> |
| <tr><td><code>druid.indexer.task.gracefulShutdownTimeout</code></td><td>Wait this long on middleManager restart for restorable tasks to gracefully exit.</td><td>PT5M</td></tr> |
| <tr><td><code>druid.indexer.task.hadoopWorkingPath</code></td><td>Temporary working directory for Hadoop tasks.</td><td><code>/tmp/druid-indexing</code></td></tr> |
| <tr><td><code>druid.indexer.task.restoreTasksOnRestart</code></td><td>If true, MiddleManagers will attempt to stop tasks gracefully on shutdown and restore them on restart.</td><td>false</td></tr> |
| <tr><td><code>druid.indexer.server.maxChatRequests</code></td><td>Maximum number of concurrent requests served by a task's chat handler. Set to 0 to disable limiting.</td><td>0</td></tr> |
| </tbody> |
| </table> |
| <p>If the peon is running in remote mode, there must be an Overlord up and running. Peons in remote mode can set the following configurations:</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.peon.taskActionClient.retry.minWait</code></td><td>The minimum retry time to communicate with Overlord.</td><td>PT5S</td></tr> |
| <tr><td><code>druid.peon.taskActionClient.retry.maxWait</code></td><td>The maximum retry time to communicate with Overlord.</td><td>PT1M</td></tr> |
| <tr><td><code>druid.peon.taskActionClient.retry.maxRetryCount</code></td><td>The maximum number of retries to communicate with Overlord.</td><td>60</td></tr> |
| </tbody> |
| </table> |
| <h5><a class="anchor" aria-hidden="true" id="segmentwriteoutmediumfactory"></a><a href="#segmentwriteoutmediumfactory" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>SegmentWriteOutMediumFactory</h5> |
| <p>When new segments are created, Druid temporarily stores some preprocessed data in some buffers. Currently three types of |
| <em>medium</em> exist for those buffers: <em>temporary files</em>, <em>off-heap memory</em>, and <em>on-heap memory</em>.</p> |
| <p><em>Temporary files</em> (<code>tmpFile</code>) are stored under the task working directory (see <code>druid.indexer.task.baseTaskDir</code> |
| configuration above) and thus share it's mounting properties, e. g. they could be backed by HDD, SSD or memory (tmpfs). |
| This type of medium may do unnecessary disk I/O and requires some disk space to be available.</p> |
| <p><em>Off-heap memory medium</em> (<code>offHeapMemory</code>) creates buffers in off-heap memory of a JVM process that is running a task. |
| This type of medium is preferred, but it may require to allow the JVM to have more off-heap memory, by changing |
| <code>-XX:MaxDirectMemorySize</code> configuration. It is not yet understood how does the required off-heap memory size relates |
| to the size of the segments being created. But definitely it doesn't make sense to add more extra off-heap memory, |
| than the configured maximum <em>heap</em> size (<code>-Xmx</code>) for the same JVM.</p> |
| <p><em>On-heap memory medium</em> (<code>onHeapMemory</code>) creates buffers using the allocated heap memory of the JVM process running a task. |
| Using on-heap memory introduces garbage collection overhead and so is not recommended in most cases. This type of medium is |
| most helpful for tasks run on external clusters where it may be difficult to allocate and work with direct memory |
| effectively.</p> |
| <p>For most types of tasks SegmentWriteOutMediumFactory could be configured per-task (see <a href="/docs/latest/ingestion/tasks.html">Tasks</a> |
| page, "TuningConfig" section), but if it's not specified for a task, or it's not supported for a particular task type, |
| then the value from the configuration below is used:</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.peon.defaultSegmentWriteOutMediumFactory.type</code></td><td><code>tmpFile</code>, <code>offHeapMemory</code>, or <code>onHeapMemory</code>, see explanation above</td><td><code>tmpFile</code></td></tr> |
| </tbody> |
| </table> |
| <h3><a class="anchor" aria-hidden="true" id="indexer"></a><a href="#indexer" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Indexer</h3> |
| <h4><a class="anchor" aria-hidden="true" id="indexer-process-configuration"></a><a href="#indexer-process-configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Indexer Process Configuration</h4> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.host</code></td><td>The host for the current process. This is used to advertise the current processes location as reachable from another process and should generally be specified such that <code>http://${druid.host}/</code> could actually talk to this process</td><td>InetAddress.getLocalHost().getCanonicalHostName()</td></tr> |
| <tr><td><code>druid.bindOnHost</code></td><td>Indicating whether the process's internal jetty server bind on <code>druid.host</code>. Default is false, which means binding to all interfaces.</td><td>false</td></tr> |
| <tr><td><code>druid.plaintextPort</code></td><td>This is the port to actually listen on; unless port mapping is used, this will be the same port as is on <code>druid.host</code></td><td>8091</td></tr> |
| <tr><td><code>druid.tlsPort</code></td><td>TLS port for HTTPS connector, if <a href="/docs/latest/operations/tls-support.html">druid.enableTlsPort</a> is set then this config will be used. If <code>druid.host</code> contains port then that port will be ignored. This should be a non-negative Integer.</td><td>8283</td></tr> |
| <tr><td><code>druid.service</code></td><td>The name of the service. This is used as a dimension when emitting metrics and alerts to differentiate between the various services</td><td>druid/indexer</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="indexer-general-configuration"></a><a href="#indexer-general-configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Indexer General Configuration</h4> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.worker.version</code></td><td>Version identifier for the Indexer.</td><td>0</td></tr> |
| <tr><td><code>druid.worker.capacity</code></td><td>Maximum number of tasks the Indexer can accept.</td><td>Number of available processors - 1</td></tr> |
| <tr><td><code>druid.worker.globalIngestionHeapLimitBytes</code></td><td>Total amount of heap available for ingestion processing. This is applied by automatically setting the <code>maxBytesInMemory</code> property on tasks.</td><td>60% of configured JVM heap</td></tr> |
| <tr><td><code>druid.worker.numConcurrentMerges</code></td><td>Maximum number of segment persist or merge operations that can run concurrently across all tasks.</td><td><code>druid.worker.capacity</code> / 2, rounded down</td></tr> |
| <tr><td><code>druid.indexer.task.baseDir</code></td><td>Base temporary working directory.</td><td><code>System.getProperty("java.io.tmpdir")</code></td></tr> |
| <tr><td><code>druid.indexer.task.baseTaskDir</code></td><td>Base temporary working directory for tasks.</td><td><code>${druid.indexer.task.baseDir}/persistent/tasks</code></td></tr> |
| <tr><td><code>druid.indexer.task.defaultHadoopCoordinates</code></td><td>Hadoop version to use with HadoopIndexTasks that do not request a particular version.</td><td>org.apache.hadoop:hadoop-client:2.8.5</td></tr> |
| <tr><td><code>druid.indexer.task.gracefulShutdownTimeout</code></td><td>Wait this long on Indexer restart for restorable tasks to gracefully exit.</td><td>PT5M</td></tr> |
| <tr><td><code>druid.indexer.task.hadoopWorkingPath</code></td><td>Temporary working directory for Hadoop tasks.</td><td><code>/tmp/druid-indexing</code></td></tr> |
| <tr><td><code>druid.indexer.task.restoreTasksOnRestart</code></td><td>If true, the Indexer will attempt to stop tasks gracefully on shutdown and restore them on restart.</td><td>false</td></tr> |
| <tr><td><code>druid.peon.taskActionClient.retry.minWait</code></td><td>The minimum retry time to communicate with Overlord.</td><td>PT5S</td></tr> |
| <tr><td><code>druid.peon.taskActionClient.retry.maxWait</code></td><td>The maximum retry time to communicate with Overlord.</td><td>PT1M</td></tr> |
| <tr><td><code>druid.peon.taskActionClient.retry.maxRetryCount</code></td><td>The maximum number of retries to communicate with Overlord.</td><td>60</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="indexer-concurrent-requests"></a><a href="#indexer-concurrent-requests" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Indexer Concurrent Requests</h4> |
| <p>Druid uses Jetty to serve HTTP requests.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.server.http.numThreads</code></td><td>Number of threads for HTTP requests. Please see the <a href="../design/indexer.html#server-http-threads">Indexer Server HTTP threads</a> documentation for more details on how the Indexer uses this configuration.</td><td>max(10, (Number of cores * 17) / 16 + 2) + 30</td></tr> |
| <tr><td><code>druid.server.http.queueSize</code></td><td>Size of the worker queue used by Jetty server to temporarily store incoming client connections. If this value is set and a request is rejected by jetty because queue is full then client would observe request failure with TCP connection being closed immediately with a completely empty response from server.</td><td>Unbounded</td></tr> |
| <tr><td><code>druid.server.http.maxIdleTime</code></td><td>The Jetty max idle time for a connection.</td><td>PT5M</td></tr> |
| <tr><td><code>druid.server.http.enableRequestLimit</code></td><td>If enabled, no requests would be queued in jetty queue and "HTTP 429 Too Many Requests" error response would be sent.</td><td>false</td></tr> |
| <tr><td><code>druid.server.http.defaultQueryTimeout</code></td><td>Query timeout in millis, beyond which unfinished queries will be cancelled</td><td>300000</td></tr> |
| <tr><td><code>druid.server.http.gracefulShutdownTimeout</code></td><td>The maximum amount of time Jetty waits after receiving shutdown signal. After this timeout the threads will be forcefully shutdown. This allows any queries that are executing to complete.</td><td><code>PT0S</code> (do not wait)</td></tr> |
| <tr><td><code>druid.server.http.unannouncePropagationDelay</code></td><td>How long to wait for zookeeper unannouncements to propagate before shutting down Jetty. This is a minimum and <code>druid.server.http.gracefulShutdownTimeout</code> does not start counting down until after this period elapses.</td><td><code>PT0S</code> (do not wait)</td></tr> |
| <tr><td><code>druid.server.http.maxQueryTimeout</code></td><td>Maximum allowed value (in milliseconds) for <code>timeout</code> parameter. See <a href="../querying/query-context.html">query-context</a> to know more about <code>timeout</code>. Query is rejected if the query context <code>timeout</code> is greater than this value.</td><td>Long.MAX_VALUE</td></tr> |
| <tr><td><code>druid.server.http.maxRequestHeaderSize</code></td><td>Maximum size of a request header in bytes. Larger headers consume more memory and can make a server more vulnerable to denial of service attacks.</td><td>8 * 1024</td></tr> |
| <tr><td><code>druid.server.http.enableForwardedRequestCustomizer</code></td><td>If enabled, adds Jetty ForwardedRequestCustomizer which reads X-Forwarded-* request headers to manipulate servlet request object when Druid is used behind a proxy.</td><td>false</td></tr> |
| <tr><td><code>druid.server.http.allowedHttpMethods</code></td><td>List of HTTP methods that should be allowed in addition to the ones required by Druid APIs. Druid APIs require GET, PUT, POST, and DELETE, which are always allowed. This option is not useful unless you have installed an extension that needs these additional HTTP methods or that adds functionality related to CORS. None of Druid's bundled extensions require these methods.</td><td>[]</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="indexer-processing-resources"></a><a href="#indexer-processing-resources" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Indexer Processing Resources</h4> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.processing.buffer.sizeBytes</code></td><td>This specifies a buffer size for the storage of intermediate results. The computation engine in the Indexer processes will use a scratch buffer of this size to do all of their intermediate computations off-heap. Larger values allow for more aggregations in a single pass over the data while smaller values can require more passes depending on the query that is being executed.</td><td>auto (max 1GB)</td></tr> |
| <tr><td><code>druid.processing.buffer.poolCacheMaxCount</code></td><td>processing buffer pool caches the buffers for later use, this is the maximum count cache will grow to. note that pool can create more buffers than it can cache if necessary.</td><td>Integer.MAX_VALUE</td></tr> |
| <tr><td><code>druid.processing.formatString</code></td><td>Indexer processes use this format string to name their processing threads.</td><td>processing-%s</td></tr> |
| <tr><td><code>druid.processing.numMergeBuffers</code></td><td>The number of direct memory buffers available for merging query results. The buffers are sized by <code>druid.processing.buffer.sizeBytes</code>. This property is effectively a concurrency limit for queries that require merging buffers. If you are using any queries that require merge buffers (currently, just groupBy v2) then you should have at least two of these.</td><td><code>max(2, druid.processing.numThreads / 4)</code></td></tr> |
| <tr><td><code>druid.processing.numThreads</code></td><td>The number of processing threads to have available for parallel processing of segments. Our rule of thumb is <code>num_cores - 1</code>, which means that even under heavy load there will still be one core available to do background tasks like talking with ZooKeeper and pulling down segments. If only one core is available, this property defaults to the value <code>1</code>.</td><td>Number of cores - 1 (or 1)</td></tr> |
| <tr><td><code>druid.processing.columnCache.sizeBytes</code></td><td>Maximum size in bytes for the dimension value lookup cache. Any value greater than <code>0</code> enables the cache. It is currently disabled by default. Enabling the lookup cache can significantly improve the performance of aggregators operating on dimension values, such as the JavaScript aggregator, or cardinality aggregator, but can slow things down if the cache hit rate is low (i.e. dimensions with few repeating values). Enabling it may also require additional garbage collection tuning to avoid long GC pauses.</td><td><code>0</code> (disabled)</td></tr> |
| <tr><td><code>druid.processing.fifo</code></td><td>If the processing queue should treat tasks of equal priority in a FIFO manner</td><td><code>false</code></td></tr> |
| <tr><td><code>druid.processing.tmpDir</code></td><td>Path where temporary files created while processing a query should be stored. If specified, this configuration takes priority over the default <code>java.io.tmpdir</code> path.</td><td>path represented by <code>java.io.tmpdir</code></td></tr> |
| </tbody> |
| </table> |
| <p>The amount of direct memory needed by Druid is at least |
| <code>druid.processing.buffer.sizeBytes * (druid.processing.numMergeBuffers + druid.processing.numThreads + 1)</code>. You can |
| ensure at least this amount of direct memory is available by providing <code>-XX:MaxDirectMemorySize=<VALUE></code> at the command |
| line.</p> |
| <h4><a class="anchor" aria-hidden="true" id="query-configurations"></a><a href="#query-configurations" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Query Configurations</h4> |
| <p>See <a href="#general-query-configuration">general query configuration</a>.</p> |
| <h4><a class="anchor" aria-hidden="true" id="indexer-caching"></a><a href="#indexer-caching" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Indexer Caching</h4> |
| <p>You can optionally configure caching to be enabled on the Indexer by setting caching configs here.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Possible Values</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.realtime.cache.useCache</code></td><td>true, false</td><td>Enable the cache on the realtime.</td><td>false</td></tr> |
| <tr><td><code>druid.realtime.cache.populateCache</code></td><td>true, false</td><td>Populate the cache on the realtime.</td><td>false</td></tr> |
| <tr><td><code>druid.realtime.cache.unCacheable</code></td><td>All druid query types</td><td>All query types to not cache.</td><td><code>[]</code></td></tr> |
| <tr><td><code>druid.realtime.cache.maxEntrySize</code></td><td>positive integer</td><td>Maximum cache entry size in bytes.</td><td>1_000_000</td></tr> |
| </tbody> |
| </table> |
| <p>See <a href="#cache-configuration">cache configuration</a> for how to configure cache settings.</p> |
| <p>Note that only local caches such as the <code>local</code>-type cache and <code>caffeine</code> cache are supported. If a remote cache such as <code>memcached</code> is used, it will be ignored.</p> |
| <h3><a class="anchor" aria-hidden="true" id="historical"></a><a href="#historical" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Historical</h3> |
| <p>For general Historical Process information, see <a href="/docs/latest/design/historical.html">here</a>.</p> |
| <p>These Historical configurations can be defined in the <code>historical/runtime.properties</code> file.</p> |
| <h4><a class="anchor" aria-hidden="true" id="historical-process-configuration"></a><a href="#historical-process-configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Historical Process Configuration</h4> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.host</code></td><td>The host for the current process. This is used to advertise the current processes location as reachable from another process and should generally be specified such that <code>http://${druid.host}/</code> could actually talk to this process</td><td>InetAddress.getLocalHost().getCanonicalHostName()</td></tr> |
| <tr><td><code>druid.bindOnHost</code></td><td>Indicating whether the process's internal jetty server bind on <code>druid.host</code>. Default is false, which means binding to all interfaces.</td><td>false</td></tr> |
| <tr><td><code>druid.plaintextPort</code></td><td>This is the port to actually listen on; unless port mapping is used, this will be the same port as is on <code>druid.host</code></td><td>8083</td></tr> |
| <tr><td><code>druid.tlsPort</code></td><td>TLS port for HTTPS connector, if <a href="/docs/latest/operations/tls-support.html">druid.enableTlsPort</a> is set then this config will be used. If <code>druid.host</code> contains port then that port will be ignored. This should be a non-negative Integer.</td><td>8283</td></tr> |
| <tr><td><code>druid.service</code></td><td>The name of the service. This is used as a dimension when emitting metrics and alerts to differentiate between the various services</td><td>druid/historical</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="historical-general-configuration"></a><a href="#historical-general-configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Historical General Configuration</h4> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.server.maxSize</code></td><td>The maximum number of bytes-worth of segments that the process wants assigned to it. The Coordinator process will attempt to assign segments to a Historical process only if this property is greater than the total size of segments served by it. Since this property defines the upper limit on the total segment size that can be assigned to a Historical, it is defaulted to the sum of all <code>maxSize</code> values specified within <code>druid.segmentCache.locations</code> property. Human-readable format is supported, see <a href="/docs/latest/configuration/human-readable-byte.html">here</a>.</td><td>Sum of <code>maxSize</code> values defined within <code>druid.segmentCache.locations</code></td></tr> |
| <tr><td><code>druid.server.tier</code></td><td>A string to name the distribution tier that the storage process belongs to. Many of the <a href="/docs/latest/operations/rule-configuration.html">rules Coordinator processes use</a> to manage segments can be keyed on tiers.</td><td><code>_default_tier</code></td></tr> |
| <tr><td><code>druid.server.priority</code></td><td>In a tiered architecture, the priority of the tier, thus allowing control over which processes are queried. Higher numbers mean higher priority. The default (no priority) works for architecture with no cross replication (tiers that have no data-storage overlap). Data centers typically have equal priority.</td><td>0</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="storing-segments"></a><a href="#storing-segments" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Storing Segments</h4> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.segmentCache.locations</code></td><td>Segments assigned to a Historical process are first stored on the local file system (in a disk cache) and then served by the Historical process. These locations define where that local cache resides. This value cannot be NULL or EMPTY. Here is an example <code>druid.segmentCache.locations=[{"path": "/mnt/druidSegments", "maxSize": "10k", "freeSpacePercent": 1.0}]</code>. "freeSpacePercent" is optional, if provided then enforces that much of free disk partition space while storing segments. But, it depends on File.getTotalSpace() and File.getFreeSpace() methods, so enable if only if they work for your File System.</td><td>none</td></tr> |
| <tr><td><code>druid.segmentCache.locationSelectorStrategy</code></td><td>The strategy used to select a location from the configured <code>druid.segmentCache.locations</code> for segment distribution. Possible values are <code>leastBytesUsed</code>, <code>roundRobin</code>, <code>random</code>, or <code>mostAvailableSize</code>.</td><td>leastBytesUsed</td></tr> |
| <tr><td><code>druid.segmentCache.deleteOnRemove</code></td><td>Delete segment files from cache once a process is no longer serving a segment.</td><td>true</td></tr> |
| <tr><td><code>druid.segmentCache.dropSegmentDelayMillis</code></td><td>How long a process delays before completely dropping segment.</td><td>30000 (30 seconds)</td></tr> |
| <tr><td><code>druid.segmentCache.infoDir</code></td><td>Historical processes keep track of the segments they are serving so that when the process is restarted they can reload the same segments without waiting for the Coordinator to reassign. This path defines where this metadata is kept. Directory will be created if needed.</td><td>${first_location}/info_dir</td></tr> |
| <tr><td><code>druid.segmentCache.announceIntervalMillis</code></td><td>How frequently to announce segments while segments are loading from cache. Set this value to zero to wait for all segments to be loaded before announcing.</td><td>5000 (5 seconds)</td></tr> |
| <tr><td><code>druid.segmentCache.numLoadingThreads</code></td><td>How many segments to drop or load concurrently from deep storage. Note that the work of loading segments involves downloading segments from deep storage, decompressing them and loading them to a memory mapped location. So the work is not all I/O Bound. Depending on CPU and network load, one could possibly increase this config to a higher value.</td><td>max(1,Number of cores / 6)</td></tr> |
| <tr><td><code>druid.segmentCache.numBootstrapThreads</code></td><td>How many segments to load concurrently during historical startup.</td><td><code>druid.segmentCache.numLoadingThreads</code></td></tr> |
| <tr><td><code>druid.segmentCache.lazyLoadOnStart</code></td><td>Whether or not to load segment columns metadata lazily during historical startup. When set to true, Historical startup time will be dramatically improved by deferring segment loading until the first time that segment takes part in a query, which will incur this cost instead. One catch is that if historical crashes while in the process of downloading and creating segment files, it is possible to end up with a corrupted segment on disk, this requires manual intervention to delete corrupted files. When the flag is set to true, historical startup would complete successfully and queries using this segment would fail at runtime.</td><td>false</td></tr> |
| <tr><td><code>druid.coordinator.loadqueuepeon.curator.numCallbackThreads</code></td><td>Number of threads for executing callback actions associated with loading or dropping of segments. One might want to increase this number when noticing clusters are lagging behind w.r.t. balancing segments across historical nodes.</td><td>2</td></tr> |
| </tbody> |
| </table> |
| <p>In <code>druid.segmentCache.locations</code>, <em>freeSpacePercent</em> was added because <em>maxSize</em> setting is only a theoretical limit and assumes that much space will always be available for storing segments. In case of any druid bug leading to unaccounted segment files left alone on disk or some other process writing stuff to disk, This check can start failing segment loading early before filling up the disk completely and leaving the host usable otherwise.</p> |
| <p>In <code>druid.segmentCache.locationSelectorStrategy</code>, one of leastBytesUsed, roundRobin, random, or mostAvailableSize could be specified to represent the strategy to distribute segments across multiple segment cache locations.</p> |
| <table> |
| <thead> |
| <tr><th>Strategy</th><th>Description</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>leastBytesUsed</code></td><td>selects a location which has least bytes used in absolute terms.</td></tr> |
| <tr><td><code>roundRobin</code></td><td>selects a location in a round robin fashion oblivious to the bytes used or the capacity.</td></tr> |
| <tr><td><code>random</code></td><td>selects a segment cache location randomly each time among the available storage locations.</td></tr> |
| <tr><td><code>mostAvailableSize</code></td><td>selects a segment cache location that has most free space among the available storage locations.</td></tr> |
| </tbody> |
| </table> |
| <p>Note that if <code>druid.segmentCache.numLoadingThreads</code> > 1, multiple threads can download different segments at the same time. In this case, with the leastBytesUsed strategy or mostAvailableSize strategy, historicals may select a sub-optimal storage location because each decision is based on a snapshot of the storage location status of when a segment is requested to download.</p> |
| <h4><a class="anchor" aria-hidden="true" id="historical-query-configs"></a><a href="#historical-query-configs" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Historical query configs</h4> |
| <h5><a class="anchor" aria-hidden="true" id="concurrent-requests"></a><a href="#concurrent-requests" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Concurrent Requests</h5> |
| <p>Druid uses Jetty to serve HTTP requests.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.server.http.numThreads</code></td><td>Number of threads for HTTP requests.</td><td>max(10, (Number of cores * 17) / 16 + 2) + 30</td></tr> |
| <tr><td><code>druid.server.http.queueSize</code></td><td>Size of the worker queue used by Jetty server to temporarily store incoming client connections. If this value is set and a request is rejected by jetty because queue is full then client would observe request failure with TCP connection being closed immediately with a completely empty response from server.</td><td>Unbounded</td></tr> |
| <tr><td><code>druid.server.http.maxIdleTime</code></td><td>The Jetty max idle time for a connection.</td><td>PT5M</td></tr> |
| <tr><td><code>druid.server.http.enableRequestLimit</code></td><td>If enabled, no requests would be queued in jetty queue and "HTTP 429 Too Many Requests" error response would be sent.</td><td>false</td></tr> |
| <tr><td><code>druid.server.http.defaultQueryTimeout</code></td><td>Query timeout in millis, beyond which unfinished queries will be cancelled</td><td>300000</td></tr> |
| <tr><td><code>druid.server.http.gracefulShutdownTimeout</code></td><td>The maximum amount of time Jetty waits after receiving shutdown signal. After this timeout the threads will be forcefully shutdown. This allows any queries that are executing to complete.</td><td><code>PT0S</code> (do not wait)</td></tr> |
| <tr><td><code>druid.server.http.unannouncePropagationDelay</code></td><td>How long to wait for zookeeper unannouncements to propagate before shutting down Jetty. This is a minimum and <code>druid.server.http.gracefulShutdownTimeout</code> does not start counting down until after this period elapses.</td><td><code>PT0S</code> (do not wait)</td></tr> |
| <tr><td><code>druid.server.http.maxQueryTimeout</code></td><td>Maximum allowed value (in milliseconds) for <code>timeout</code> parameter. See <a href="/docs/latest/querying/query-context.html">query-context</a> to know more about <code>timeout</code>. Query is rejected if the query context <code>timeout</code> is greater than this value.</td><td>Long.MAX_VALUE</td></tr> |
| <tr><td><code>druid.server.http.maxRequestHeaderSize</code></td><td>Maximum size of a request header in bytes. Larger headers consume more memory and can make a server more vulnerable to denial of service attacks.</td><td>8 * 1024</td></tr> |
| </tbody> |
| </table> |
| <h5><a class="anchor" aria-hidden="true" id="processing"></a><a href="#processing" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Processing</h5> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.processing.buffer.sizeBytes</code></td><td>This specifies a buffer size for the storage of intermediate results. The computation engine in both the Historical and Realtime processes will use a scratch buffer of this size to do all of their intermediate computations off-heap. Larger values allow for more aggregations in a single pass over the data while smaller values can require more passes depending on the query that is being executed.</td><td>auto (max 1GB)</td></tr> |
| <tr><td><code>druid.processing.buffer.poolCacheMaxCount</code></td><td>processing buffer pool caches the buffers for later use, this is the maximum count cache will grow to. note that pool can create more buffers than it can cache if necessary.</td><td>Integer.MAX_VALUE</td></tr> |
| <tr><td><code>druid.processing.formatString</code></td><td>Realtime and Historical processes use this format string to name their processing threads.</td><td>processing-%s</td></tr> |
| <tr><td><code>druid.processing.numMergeBuffers</code></td><td>The number of direct memory buffers available for merging query results. The buffers are sized by <code>druid.processing.buffer.sizeBytes</code>. This property is effectively a concurrency limit for queries that require merging buffers. If you are using any queries that require merge buffers (currently, just groupBy v2) then you should have at least two of these.</td><td><code>max(2, druid.processing.numThreads / 4)</code></td></tr> |
| <tr><td><code>druid.processing.numThreads</code></td><td>The number of processing threads to have available for parallel processing of segments. Our rule of thumb is <code>num_cores - 1</code>, which means that even under heavy load there will still be one core available to do background tasks like talking with ZooKeeper and pulling down segments. If only one core is available, this property defaults to the value <code>1</code>.</td><td>Number of cores - 1 (or 1)</td></tr> |
| <tr><td><code>druid.processing.columnCache.sizeBytes</code></td><td>Maximum size in bytes for the dimension value lookup cache. Any value greater than <code>0</code> enables the cache. It is currently disabled by default. Enabling the lookup cache can significantly improve the performance of aggregators operating on dimension values, such as the JavaScript aggregator, or cardinality aggregator, but can slow things down if the cache hit rate is low (i.e. dimensions with few repeating values). Enabling it may also require additional garbage collection tuning to avoid long GC pauses.</td><td><code>0</code> (disabled)</td></tr> |
| <tr><td><code>druid.processing.fifo</code></td><td>If the processing queue should treat tasks of equal priority in a FIFO manner</td><td><code>false</code></td></tr> |
| <tr><td><code>druid.processing.tmpDir</code></td><td>Path where temporary files created while processing a query should be stored. If specified, this configuration takes priority over the default <code>java.io.tmpdir</code> path.</td><td>path represented by <code>java.io.tmpdir</code></td></tr> |
| </tbody> |
| </table> |
| <p>The amount of direct memory needed by Druid is at least |
| <code>druid.processing.buffer.sizeBytes * (druid.processing.numMergeBuffers + druid.processing.numThreads + 1)</code>. You can |
| ensure at least this amount of direct memory is available by providing <code>-XX:MaxDirectMemorySize=<VALUE></code> at the command |
| line.</p> |
| <h5><a class="anchor" aria-hidden="true" id="historical-query-configuration"></a><a href="#historical-query-configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Historical query configuration</h5> |
| <p>See <a href="#general-query-configuration">general query configuration</a>.</p> |
| <h4><a class="anchor" aria-hidden="true" id="historical-caching"></a><a href="#historical-caching" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Historical Caching</h4> |
| <p>You can optionally only configure caching to be enabled on the Historical by setting caching configs here.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Possible Values</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.historical.cache.useCache</code></td><td>true, false</td><td>Enable the cache on the Historical.</td><td>false</td></tr> |
| <tr><td><code>druid.historical.cache.populateCache</code></td><td>true, false</td><td>Populate the cache on the Historical.</td><td>false</td></tr> |
| <tr><td><code>druid.historical.cache.unCacheable</code></td><td>All druid query types</td><td>All query types to not cache.</td><td><code>[]</code></td></tr> |
| <tr><td><code>druid.historical.cache.maxEntrySize</code></td><td>Maximum cache entry size in bytes.</td><td>1_000_000</td></tr> |
| </tbody> |
| </table> |
| <p>See <a href="#cache-configuration">cache configuration</a> for how to configure cache settings.</p> |
| <h2><a class="anchor" aria-hidden="true" id="query-server"></a><a href="#query-server" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Query Server</h2> |
| <p>This section contains the configuration options for the processes that reside on Query servers (Brokers) in the suggested <a href="../design/processes.html#server-types">three-server configuration</a>.</p> |
| <p>Configuration options for the experimental <a href="/docs/latest/design/router.html">Router process</a> are also provided here.</p> |
| <h3><a class="anchor" aria-hidden="true" id="broker"></a><a href="#broker" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Broker</h3> |
| <p>For general Broker process information, see <a href="/docs/latest/design/broker.html">here</a>.</p> |
| <p>These Broker configurations can be defined in the <code>broker/runtime.properties</code> file.</p> |
| <h4><a class="anchor" aria-hidden="true" id="broker-process-configs"></a><a href="#broker-process-configs" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Broker Process Configs</h4> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.host</code></td><td>The host for the current process. This is used to advertise the current processes location as reachable from another process and should generally be specified such that <code>http://${druid.host}/</code> could actually talk to this process</td><td>InetAddress.getLocalHost().getCanonicalHostName()</td></tr> |
| <tr><td><code>druid.bindOnHost</code></td><td>Indicating whether the process's internal jetty server bind on <code>druid.host</code>. Default is false, which means binding to all interfaces.</td><td>false</td></tr> |
| <tr><td><code>druid.plaintextPort</code></td><td>This is the port to actually listen on; unless port mapping is used, this will be the same port as is on <code>druid.host</code></td><td>8082</td></tr> |
| <tr><td><code>druid.tlsPort</code></td><td>TLS port for HTTPS connector, if <a href="/docs/latest/operations/tls-support.html">druid.enableTlsPort</a> is set then this config will be used. If <code>druid.host</code> contains port then that port will be ignored. This should be a non-negative Integer.</td><td>8282</td></tr> |
| <tr><td><code>druid.service</code></td><td>The name of the service. This is used as a dimension when emitting metrics and alerts to differentiate between the various services</td><td>druid/broker</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="query-configuration"></a><a href="#query-configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Query configuration</h4> |
| <h5><a class="anchor" aria-hidden="true" id="query-routing"></a><a href="#query-routing" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Query routing</h5> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Possible Values</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.broker.balancer.type</code></td><td><code>random</code>, <code>connectionCount</code></td><td>Determines how the broker balances connections to Historical processes. <code>random</code> choose randomly, <code>connectionCount</code> picks the process with the fewest number of active connections to</td><td><code>random</code></td></tr> |
| <tr><td><code>druid.broker.select.tier</code></td><td><code>highestPriority</code>, <code>lowestPriority</code>, <code>custom</code></td><td>If segments are cross-replicated across tiers in a cluster, you can tell the broker to prefer to select segments in a tier with a certain priority.</td><td><code>highestPriority</code></td></tr> |
| <tr><td><code>druid.broker.select.tier.custom.priorities</code></td><td><code>An array of integer priorities.</code> E.g., <code>[-1, 0, 1, 2]</code></td><td>Select servers in tiers with a custom priority list.</td><td>The config only has effect if <code>druid.broker.select.tier</code> is set to <code>custom</code>. If <code>druid.broker.select.tier</code> is set to <code>custom</code> but this config is not specified, the effect is the same as <code>druid.broker.select.tier</code> set to <code>highestPriority</code>. Any of the integers in this config can be ignored if there's no corresponding tiers with such priorities. Tiers with priorities explicitly specified in this config always have higher priority than those not and those not specified fall back to use <code>highestPriority</code> strategy among themselves.</td></tr> |
| </tbody> |
| </table> |
| <h5><a class="anchor" aria-hidden="true" id="query-prioritization-and-laning"></a><a href="#query-prioritization-and-laning" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Query prioritization and laning</h5> |
| <p><em>Laning strategies</em> allow you to control capacity utilization for heterogeneous query workloads. With laning, the broker examines and classifies a query for the purpose of assigning it to a 'lane'. Lanes have capacity limits, enforced by the broker, that can be used to ensure sufficient resources are available for other lanes or for interactive queries (with no lane), or to limit overall throughput for queries within the lane. Requests in excess of the capacity are discarded with an HTTP 429 status code.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.query.scheduler.numThreads</code></td><td>Maximum number of HTTP threads to dedicate to query processing. To save HTTP thread capacity, this should be lower than <code>druid.server.http.numThreads</code>, but it is worth noting that like <code>druid.server.http.enableRequestLimit</code> is set that query requests over this limit will be denied instead of waiting in the Jetty HTTP request queue.</td><td>Unbounded</td></tr> |
| <tr><td><code>druid.query.scheduler.laning.strategy</code></td><td>Query laning strategy to use to assign queries to a lane in order to control capacities for certain classes of queries.</td><td><code>none</code></td></tr> |
| <tr><td><code>druid.query.scheduler.prioritization.strategy</code></td><td>Query prioritization strategy to automatically assign priorities.</td><td><code>manual</code></td></tr> |
| </tbody> |
| </table> |
| <h5><a class="anchor" aria-hidden="true" id="prioritization-strategies"></a><a href="#prioritization-strategies" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Prioritization strategies</h5> |
| <h6><a class="anchor" aria-hidden="true" id="manual-prioritization-strategy"></a><a href="#manual-prioritization-strategy" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Manual prioritization strategy</h6> |
| <p>With this configuration, queries are never assigned a priority automatically, but will preserve a priority manually set on the <a href="/docs/latest/querying/query-context.html">query context</a> with the <code>priority</code> key. This mode can be explicitly set by setting <code>druid.query.scheduler.prioritization.strategy</code> to <code>none</code>.</p> |
| <h6><a class="anchor" aria-hidden="true" id="threshold-prioritization-strategy"></a><a href="#threshold-prioritization-strategy" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Threshold prioritization strategy</h6> |
| <p>This prioritization strategy lowers the priority of queries that cross any of a configurable set of thresholds, such as how far in the past the data is, how large of an interval a query covers, or the number of segments taking part in a query.</p> |
| <p>This strategy can be enabled by setting <code>druid.query.scheduler.prioritization.strategy</code> to <code>threshold</code>.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.query.scheduler.prioritization.periodThreshold</code></td><td>ISO duration threshold for how old data can be queried before automatically adjusting query priority.</td><td>None</td></tr> |
| <tr><td><code>druid.query.scheduler.prioritization.durationThreshold</code></td><td>ISO duration threshold for maximum duration a queries interval can span before the priority is automatically adjusted.</td><td>None</td></tr> |
| <tr><td><code>druid.query.scheduler.prioritization.segmentCountThreshold</code></td><td>Number threshold for maximum number of segments that can take part in a query before its priority is automatically adjusted.</td><td>None</td></tr> |
| <tr><td><code>druid.query.scheduler.prioritization.adjustment</code></td><td>Amount to reduce the priority of queries which cross any threshold.</td><td>None</td></tr> |
| </tbody> |
| </table> |
| <h5><a class="anchor" aria-hidden="true" id="laning-strategies"></a><a href="#laning-strategies" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Laning strategies</h5> |
| <h6><a class="anchor" aria-hidden="true" id="no-laning-strategy"></a><a href="#no-laning-strategy" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>No laning strategy</h6> |
| <p>In this mode, queries are never assigned a lane, and the concurrent query count will only be limited by <code>druid.server.http.numThreads</code> or <code>druid.query.scheduler.numThreads</code>, if set. This is the default Druid query scheduler operating mode. Enable this strategy explicitly by setting <code>druid.query.scheduler.laning.strategy</code> to <code>none</code>.</p> |
| <h6><a class="anchor" aria-hidden="true" id="highlow-laning-strategy"></a><a href="#highlow-laning-strategy" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>'High/Low' laning strategy</h6> |
| <p>This laning strategy splits queries with a <code>priority</code> below zero into a <code>low</code> query lane, automatically. Queries with priority of zero (the default) or above are considered 'interactive'. The limit on <code>low</code> queries can be set to some desired percentage of the total capacity (or HTTP thread pool size), reserving capacity for interactive queries. Queries in the <code>low</code> lane are <em>not</em> guaranteed their capacity, which may be consumed by interactive queries, but may use up to this limit if total capacity is available.</p> |
| <p>If the <code>low</code> lane is specified in the <a href="/docs/latest/querying/query-context.html">query context</a> <code>lane</code> parameter, this will override the computed lane.</p> |
| <p>This strategy can be enabled by setting <code>druid.query.scheduler.laning.strategy=hilo</code>.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.query.scheduler.laning.maxLowPercent</code></td><td>Maximum percent of the smaller number of <code>druid.server.http.numThreads</code> or <code>druid.query.scheduler.numThreads</code>, defining the number of HTTP threads that can be used by queries with a priority lower than 0. Value must be an integer in the range 1 to 100, and will be rounded up</td><td>No default, must be set if using this mode</td></tr> |
| </tbody> |
| </table> |
| <h6><a class="anchor" aria-hidden="true" id="manual-laning-strategy"></a><a href="#manual-laning-strategy" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>'Manual' laning strategy</h6> |
| <p>This laning strategy is best suited for cases where one or more external applications which query Druid are capable of manually deciding what lane a given query should belong to. Configured with a map of lane names to percent or exact max capacities, queries with a matching <code>lane</code> parameter in the <a href="/docs/latest/querying/query-context.html">query context</a> will be subjected to those limits.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.query.scheduler.laning.lanes.{name}</code></td><td>Maximum percent or exact limit of queries that can concurrently run in the defined lanes. Any number of lanes may be defined like this. The lane names 'total' and 'default' are reserved for internal use.</td><td>No default, must define at least one lane with a limit above 0. If <code>druid.query.scheduler.laning.isLimitPercent</code> is set to <code>true</code>, values must be integers in the range of 1 to 100.</td></tr> |
| <tr><td><code>druid.query.scheduler.laning.isLimitPercent</code></td><td>If set to <code>true</code>, the values set for <code>druid.query.scheduler.laning.lanes</code> will be treated as a percent of the smaller number of <code>druid.server.http.numThreads</code> or <code>druid.query.scheduler.numThreads</code>. Note that in this mode, these lane values across lanes are <em>not</em> required to add up to, and can exceed, 100%.</td><td><code>false</code></td></tr> |
| </tbody> |
| </table> |
| <h5><a class="anchor" aria-hidden="true" id="server-configuration"></a><a href="#server-configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Server Configuration</h5> |
| <p>Druid uses Jetty to serve HTTP requests. Each query being processed consumes a single thread from <code>druid.server.http.numThreads</code>, so consider defining <code>druid.query.scheduler.numThreads</code> to a lower value in order to reserve HTTP threads for responding to health checks, lookup loading, and other non-query, and in most cases comparatively very short lived, HTTP requests.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.server.http.numThreads</code></td><td>Number of threads for HTTP requests.</td><td>max(10, (Number of cores * 17) / 16 + 2) + 30</td></tr> |
| <tr><td><code>druid.server.http.queueSize</code></td><td>Size of the worker queue used by Jetty server to temporarily store incoming client connections. If this value is set and a request is rejected by jetty because queue is full then client would observe request failure with TCP connection being closed immediately with a completely empty response from server.</td><td>Unbounded</td></tr> |
| <tr><td><code>druid.server.http.maxIdleTime</code></td><td>The Jetty max idle time for a connection.</td><td>PT5M</td></tr> |
| <tr><td><code>druid.server.http.enableRequestLimit</code></td><td>If enabled, no requests would be queued in jetty queue and "HTTP 429 Too Many Requests" error response would be sent.</td><td>false</td></tr> |
| <tr><td><code>druid.server.http.defaultQueryTimeout</code></td><td>Query timeout in millis, beyond which unfinished queries will be cancelled</td><td>300000</td></tr> |
| <tr><td><code>druid.server.http.maxScatterGatherBytes</code></td><td>Maximum number of bytes gathered from data processes such as Historicals and realtime processes to execute a query. Queries that exceed this limit will fail. This is an advance configuration that allows to protect in case Broker is under heavy load and not utilizing the data gathered in memory fast enough and leading to OOMs. This limit can be further reduced at query time using <code>maxScatterGatherBytes</code> in the context. Note that having large limit is not necessarily bad if broker is never under heavy concurrent load in which case data gathered is processed quickly and freeing up the memory used. Human-readable format is supported, see <a href="/docs/latest/configuration/human-readable-byte.html">here</a>.</td><td>Long.MAX_VALUE</td></tr> |
| <tr><td><code>druid.server.http.maxSubqueryRows</code></td><td>Maximum number of rows from subqueries per query. These rows are stored in memory.</td><td>100000</td></tr> |
| <tr><td><code>druid.server.http.gracefulShutdownTimeout</code></td><td>The maximum amount of time Jetty waits after receiving shutdown signal. After this timeout the threads will be forcefully shutdown. This allows any queries that are executing to complete.</td><td><code>PT0S</code> (do not wait)</td></tr> |
| <tr><td><code>druid.server.http.unannouncePropagationDelay</code></td><td>How long to wait for zookeeper unannouncements to propagate before shutting down Jetty. This is a minimum and <code>druid.server.http.gracefulShutdownTimeout</code> does not start counting down until after this period elapses.</td><td><code>PT0S</code> (do not wait)</td></tr> |
| <tr><td><code>druid.server.http.maxQueryTimeout</code></td><td>Maximum allowed value (in milliseconds) for <code>timeout</code> parameter. See <a href="/docs/latest/querying/query-context.html">query-context</a> to know more about <code>timeout</code>. Query is rejected if the query context <code>timeout</code> is greater than this value.</td><td>Long.MAX_VALUE</td></tr> |
| <tr><td><code>druid.server.http.maxRequestHeaderSize</code></td><td>Maximum size of a request header in bytes. Larger headers consume more memory and can make a server more vulnerable to denial of service attacks.</td><td>8 * 1024</td></tr> |
| </tbody> |
| </table> |
| <h5><a class="anchor" aria-hidden="true" id="client-configuration"></a><a href="#client-configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Client Configuration</h5> |
| <p>Druid Brokers use an HTTP client to communicate with with data servers (Historical servers and real-time tasks). This |
| client has the following configuration options.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.broker.http.numConnections</code></td><td>Size of connection pool for the Broker to connect to Historical and real-time processes. If there are more queries than this number that all need to speak to the same process, then they will queue up.</td><td><code>20</code></td></tr> |
| <tr><td><code>druid.broker.http.compressionCodec</code></td><td>Compression codec the Broker uses to communicate with Historical and real-time processes. May be "gzip" or "identity".</td><td><code>gzip</code></td></tr> |
| <tr><td><code>druid.broker.http.readTimeout</code></td><td>The timeout for data reads from Historical servers and real-time tasks.</td><td><code>PT15M</code></td></tr> |
| <tr><td><code>druid.broker.http.unusedConnectionTimeout</code></td><td>The timeout for idle connections in connection pool. The connection in the pool will be closed after this timeout and a new one will be established. This timeout should be less than <code>druid.broker.http.readTimeout</code>. Set this timeout = ~90% of <code>druid.broker.http.readTimeout</code></td><td><code>PT4M</code></td></tr> |
| <tr><td><code>druid.broker.http.maxQueuedBytes</code></td><td>Maximum number of bytes queued per query before exerting backpressure on the channel to the data server. Similar to <code>druid.server.http.maxScatterGatherBytes</code>, except unlike that configuration, this one will trigger backpressure rather than query failure. Zero means disabled. Can be overridden by the <a href="/docs/latest/querying/query-context.html">"maxQueuedBytes" query context parameter</a>. Human-readable format is supported, see <a href="/docs/latest/configuration/human-readable-byte.html">here</a>.</td><td><code>0</code> (disabled)</td></tr> |
| <tr><td><code>druid.broker.http.numMaxThreads</code></td><td>`Maximum number of I/O worker threads</td><td>max(10, ((number of cores * 17) / 16 + 2) + 30)`</td></tr> |
| </tbody> |
| </table> |
| <h5><a class="anchor" aria-hidden="true" id="retry-policy"></a><a href="#retry-policy" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Retry Policy</h5> |
| <p>Druid broker can optionally retry queries internally for transient errors.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.broker.retryPolicy.numTries</code></td><td>Number of tries.</td><td>1</td></tr> |
| </tbody> |
| </table> |
| <h5><a class="anchor" aria-hidden="true" id="processing-1"></a><a href="#processing-1" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Processing</h5> |
| <p>The broker uses processing configs for nested groupBy queries.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.processing.buffer.sizeBytes</code></td><td>This specifies a buffer size for the storage of intermediate results. The computation engine in both the Historical and Realtime processes will use a scratch buffer of this size to do all of their intermediate computations off-heap. Larger values allow for more aggregations in a single pass over the data while smaller values can require more passes depending on the query that is being executed. It can be configured as described in <a href="/docs/latest/configuration/human-readable-byte.html">here</a>.</td><td>auto (max 1GiB)</td></tr> |
| <tr><td><code>druid.processing.buffer.poolCacheMaxCount</code></td><td>processing buffer pool caches the buffers for later use, this is the maximum count cache will grow to. note that pool can create more buffers than it can cache if necessary.</td><td>Integer.MAX_VALUE</td></tr> |
| <tr><td><code>druid.processing.formatString</code></td><td>Realtime and Historical processes use this format string to name their processing threads.</td><td>processing-%s</td></tr> |
| <tr><td><code>druid.processing.numMergeBuffers</code></td><td>The number of direct memory buffers available for merging query results. The buffers are sized by <code>druid.processing.buffer.sizeBytes</code>. This property is effectively a concurrency limit for queries that require merging buffers. If you are using any queries that require merge buffers (currently, just groupBy v2) then you should have at least two of these.</td><td><code>max(2, druid.processing.numThreads / 4)</code></td></tr> |
| <tr><td><code>druid.processing.numThreads</code></td><td>The number of processing threads to have available for parallel processing of segments. Our rule of thumb is <code>num_cores - 1</code>, which means that even under heavy load there will still be one core available to do background tasks like talking with ZooKeeper and pulling down segments. If only one core is available, this property defaults to the value <code>1</code>.</td><td>Number of cores - 1 (or 1)</td></tr> |
| <tr><td><code>druid.processing.columnCache.sizeBytes</code></td><td>Maximum size in bytes for the dimension value lookup cache. Any value greater than <code>0</code> enables the cache. It is currently disabled by default. Enabling the lookup cache can significantly improve the performance of aggregators operating on dimension values, such as the JavaScript aggregator, or cardinality aggregator, but can slow things down if the cache hit rate is low (i.e. dimensions with few repeating values). Enabling it may also require additional garbage collection tuning to avoid long GC pauses.</td><td><code>0</code> (disabled)</td></tr> |
| <tr><td><code>druid.processing.fifo</code></td><td>If the processing queue should treat tasks of equal priority in a FIFO manner</td><td><code>false</code></td></tr> |
| <tr><td><code>druid.processing.tmpDir</code></td><td>Path where temporary files created while processing a query should be stored. If specified, this configuration takes priority over the default <code>java.io.tmpdir</code> path.</td><td>path represented by <code>java.io.tmpdir</code></td></tr> |
| <tr><td><code>druid.processing.merge.useParallelMergePool</code></td><td>Enable automatic parallel merging for Brokers on a dedicated async ForkJoinPool. If <code>false</code>, instead merges will be done serially on the <code>HTTP</code> thread pool.</td><td><code>true</code></td></tr> |
| <tr><td><code>druid.processing.merge.pool.parallelism</code></td><td>Size of ForkJoinPool. Note that the default configuration assumes that the value returned by <code>Runtime.getRuntime().availableProcessors()</code> represents 2 hyper-threads per physical core, and multiplies this value by <code>0.75</code> in attempt to size <code>1.5</code> times the number of <em>physical</em> cores.</td><td><code>Runtime.getRuntime().availableProcessors() * 0.75</code> (rounded up)</td></tr> |
| <tr><td><code>druid.processing.merge.pool.defaultMaxQueryParallelism</code></td><td>Default maximum number of parallel merge tasks per query. Note that the default configuration assumes that the value returned by <code>Runtime.getRuntime().availableProcessors()</code> represents 2 hyper-threads per physical core, and multiplies this value by <code>0.5</code> in attempt to size to the number of <em>physical</em> cores.</td><td><code>Runtime.getRuntime().availableProcessors() * 0.5</code> (rounded up)</td></tr> |
| <tr><td><code>druid.processing.merge.pool.awaitShutdownMillis</code></td><td>Time to wait for merge ForkJoinPool tasks to complete before ungracefully stopping on process shutdown in milliseconds.</td><td><code>60_000</code></td></tr> |
| <tr><td><code>druid.processing.merge.task.targetRunTimeMillis</code></td><td>Ideal run-time of each ForkJoinPool merge task, before forking off a new task to continue merging sequences.</td><td><code>100</code></td></tr> |
| <tr><td><code>druid.processing.merge.task.initialYieldNumRows</code></td><td>Number of rows to yield per ForkJoinPool merge task, before forking off a new task to continue merging sequences.</td><td><code>16384</code></td></tr> |
| <tr><td><code>druid.processing.merge.task.smallBatchNumRows</code></td><td>Size of result batches to operate on in ForkJoinPool merge tasks.</td><td><code>4096</code></td></tr> |
| </tbody> |
| </table> |
| <p>The amount of direct memory needed by Druid is at least |
| <code>druid.processing.buffer.sizeBytes * (druid.processing.numMergeBuffers + druid.processing.numThreads + 1)</code>. You can |
| ensure at least this amount of direct memory is available by providing <code>-XX:MaxDirectMemorySize=<VALUE></code> at the command |
| line.</p> |
| <h5><a class="anchor" aria-hidden="true" id="broker-query-configuration"></a><a href="#broker-query-configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Broker query configuration</h5> |
| <p>See <a href="#general-query-configuration">general query configuration</a>.</p> |
| <h4><a class="anchor" aria-hidden="true" id="sql"></a><a href="#sql" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>SQL</h4> |
| <p>The Druid SQL server is configured through the following properties on the Broker.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.sql.enable</code></td><td>Whether to enable SQL at all, including background metadata fetching. If false, this overrides all other SQL-related properties and disables SQL metadata, serving, and planning completely.</td><td>true</td></tr> |
| <tr><td><code>druid.sql.avatica.enable</code></td><td>Whether to enable JDBC querying at <code>/druid/v2/sql/avatica/</code>.</td><td>true</td></tr> |
| <tr><td><code>druid.sql.avatica.maxConnections</code></td><td>Maximum number of open connections for the Avatica server. These are not HTTP connections, but are logical client connections that may span multiple HTTP connections.</td><td>25</td></tr> |
| <tr><td><code>druid.sql.avatica.maxRowsPerFrame</code></td><td>Maximum number of rows to return in a single JDBC frame. Setting this property to -1 indicates that no row limit should be applied. Clients can optionally specify a row limit in their requests; if a client specifies a row limit, the lesser value of the client-provided limit and <code>maxRowsPerFrame</code> will be used.</td><td>5,000</td></tr> |
| <tr><td><code>druid.sql.avatica.maxStatementsPerConnection</code></td><td>Maximum number of simultaneous open statements per Avatica client connection.</td><td>4</td></tr> |
| <tr><td><code>druid.sql.avatica.connectionIdleTimeout</code></td><td>Avatica client connection idle timeout.</td><td>PT5M</td></tr> |
| <tr><td><code>druid.sql.http.enable</code></td><td>Whether to enable JSON over HTTP querying at <code>/druid/v2/sql/</code>.</td><td>true</td></tr> |
| <tr><td><code>druid.sql.planner.maxTopNLimit</code></td><td>Maximum threshold for a <a href="/docs/latest/querying/topnquery.html">TopN query</a>. Higher limits will be planned as <a href="/docs/latest/querying/groupbyquery.html">GroupBy queries</a> instead.</td><td>100000</td></tr> |
| <tr><td><code>druid.sql.planner.metadataRefreshPeriod</code></td><td>Throttle for metadata refreshes.</td><td>PT1M</td></tr> |
| <tr><td><code>druid.sql.planner.useApproximateCountDistinct</code></td><td>Whether to use an approximate cardinality algorithm for <code>COUNT(DISTINCT foo)</code>.</td><td>true</td></tr> |
| <tr><td><code>druid.sql.planner.useApproximateTopN</code></td><td>Whether to use approximate <a href="../querying/topnquery.html">TopN queries</a> when a SQL query could be expressed as such. If false, exact <a href="../querying/groupbyquery.html">GroupBy queries</a> will be used instead.</td><td>true</td></tr> |
| <tr><td><code>druid.sql.planner.requireTimeCondition</code></td><td>Whether to require SQL to have filter conditions on __time column so that all generated native queries will have user specified intervals. If true, all queries without filter condition on __time column will fail</td><td>false</td></tr> |
| <tr><td><code>druid.sql.planner.sqlTimeZone</code></td><td>Sets the default time zone for the server, which will affect how time functions and timestamp literals behave. Should be a time zone name like "America/Los_Angeles" or offset like "-08:00".</td><td>UTC</td></tr> |
| <tr><td><code>druid.sql.planner.metadataSegmentCacheEnable</code></td><td>Whether to keep a cache of published segments in broker. If true, broker polls coordinator in background to get segments from metadata store and maintains a local cache. If false, coordinator's REST API will be invoked when broker needs published segments info.</td><td>false</td></tr> |
| <tr><td><code>druid.sql.planner.metadataSegmentPollPeriod</code></td><td>How often to poll coordinator for published segments list if <code>druid.sql.planner.metadataSegmentCacheEnable</code> is set to true. Poll period is in milliseconds.</td><td>60000</td></tr> |
| </tbody> |
| </table> |
| <blockquote> |
| <p>Previous versions of Druid had properties named <code>druid.sql.planner.maxQueryCount</code> and <code>druid.sql.planner.maxSemiJoinRowsInMemory</code>. |
| These properties are no longer available. Since Druid 0.18.0, you can use <code>druid.server.http.maxSubqueryRows</code> to control the maximum |
| number of rows permitted across all subqueries.</p> |
| </blockquote> |
| <h4><a class="anchor" aria-hidden="true" id="broker-caching"></a><a href="#broker-caching" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Broker Caching</h4> |
| <p>You can optionally only configure caching to be enabled on the Broker by setting caching configs here.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Possible Values</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.broker.cache.useCache</code></td><td>true, false</td><td>Enable the cache on the Broker.</td><td>false</td></tr> |
| <tr><td><code>druid.broker.cache.populateCache</code></td><td>true, false</td><td>Populate the cache on the Broker.</td><td>false</td></tr> |
| <tr><td><code>druid.broker.cache.useResultLevelCache</code></td><td>true, false</td><td>Enable result level caching on the Broker.</td><td>false</td></tr> |
| <tr><td><code>druid.broker.cache.populateResultLevelCache</code></td><td>true, false</td><td>Populate the result level cache on the Broker.</td><td>false</td></tr> |
| <tr><td><code>druid.broker.cache.resultLevelCacheLimit</code></td><td>positive integer</td><td>Maximum size of query response that can be cached.</td><td><code>Integer.MAX_VALUE</code></td></tr> |
| <tr><td><code>druid.broker.cache.unCacheable</code></td><td>All druid query types</td><td>All query types to not cache.</td><td><code>[]</code></td></tr> |
| <tr><td><code>druid.broker.cache.cacheBulkMergeLimit</code></td><td>positive integer or 0</td><td>Queries with more segments than this number will not attempt to fetch from cache at the broker level, leaving potential caching fetches (and cache result merging) to the Historicals</td><td><code>Integer.MAX_VALUE</code></td></tr> |
| <tr><td><code>druid.broker.cache.maxEntrySize</code></td><td>positive integer</td><td>Maximum cache entry size in bytes.</td><td>1_000_000</td></tr> |
| </tbody> |
| </table> |
| <p>See <a href="#cache-configuration">cache configuration</a> for how to configure cache settings.</p> |
| <h4><a class="anchor" aria-hidden="true" id="segment-discovery"></a><a href="#segment-discovery" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Segment Discovery</h4> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Possible Values</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.serverview.type</code></td><td>batch or http</td><td>Segment discovery method to use. "http" enables discovering segments using HTTP instead of zookeeper.</td><td>batch</td></tr> |
| <tr><td><code>druid.broker.segment.watchedTiers</code></td><td>List of strings</td><td>Broker watches the segment announcements from processes serving segments to build cache of which process is serving which segments, this configuration allows to only consider segments being served from a whitelist of tiers. By default, Broker would consider all tiers. This can be used to partition your dataSources in specific Historical tiers and configure brokers in partitions so that they are only queryable for specific dataSources.</td><td>none</td></tr> |
| <tr><td><code>druid.broker.segment.watchedDataSources</code></td><td>List of strings</td><td>Broker watches the segment announcements from processes serving segments to build cache of which process is serving which segments, this configuration allows to only consider segments being served from a whitelist of dataSources. By default, Broker would consider all datasources. This can be used to configure brokers in partitions so that they are only queryable for specific dataSources.</td><td>none</td></tr> |
| <tr><td><code>druid.broker.segment.awaitInitializationOnStart</code></td><td>Boolean</td><td>Whether the Broker will wait for its view of segments to fully initialize before starting up. If set to 'true', the Broker's HTTP server will not start up, and the Broker will not announce itself as available, until the server view is initialized. See also <code>druid.sql.planner.awaitInitializationOnStart</code>, a related setting.</td><td>true</td></tr> |
| </tbody> |
| </table> |
| <h2><a class="anchor" aria-hidden="true" id="cache-configuration"></a><a href="#cache-configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Cache Configuration</h2> |
| <p>This section describes caching configuration that is common to Broker, Historical, and MiddleManager/Peon processes.</p> |
| <p>Caching could optionally be enabled on the Broker, Historical, and MiddleManager/Peon processes. See |
| <a href="#broker-caching">Broker</a>, <a href="#historical-caching">Historical</a>, and <a href="#peon-caching">Peon</a> configuration options for how to |
| enable it for different processes.</p> |
| <p>Druid uses a local in-memory cache by default, unless a different type of cache is specified. |
| Use the <code>druid.cache.type</code> configuration to set a different kind of cache.</p> |
| <p>Cache settings are set globally, so the same configuration can be re-used |
| for both Broker and Historical processes, when defined in the common properties file.</p> |
| <h3><a class="anchor" aria-hidden="true" id="cache-type"></a><a href="#cache-type" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Cache Type</h3> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Possible Values</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.cache.type</code></td><td><code>local</code>, <code>memcached</code>, <code>hybrid</code>, <code>caffeine</code></td><td>The type of cache to use for queries. See below of the configuration options for each cache type</td><td><code>caffeine</code></td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="local-cache"></a><a href="#local-cache" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Local Cache</h4> |
| <blockquote> |
| <p>DEPRECATED: Use caffeine (default as of v0.12.0) instead</p> |
| </blockquote> |
| <p>The local cache is deprecated in favor of the Caffeine cache, and may be removed in a future version of Druid. The Caffeine cache affords significantly better performance and control over eviction behavior compared to <code>local</code> cache, and is recommended in any situation where you are using JRE 8u60 or higher.</p> |
| <p>A simple in-memory LRU cache. Local cache resides in JVM heap memory, so if you enable it, make sure you increase heap size accordingly.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.cache.sizeInBytes</code></td><td>Maximum cache size in bytes. Zero disables caching.</td><td>0</td></tr> |
| <tr><td><code>druid.cache.initialSize</code></td><td>Initial size of the hashtable backing the cache.</td><td>500000</td></tr> |
| <tr><td><code>druid.cache.logEvictionCount</code></td><td>If non-zero, log cache eviction every <code>logEvictionCount</code> items.</td><td>0</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="caffeine-cache"></a><a href="#caffeine-cache" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Caffeine Cache</h4> |
| <p>A highly performant local cache implementation for Druid based on <a href="https://github.com/ben-manes/caffeine">Caffeine</a>. Requires a JRE8u60 or higher if using <code>COMMON_FJP</code>.</p> |
| <h5><a class="anchor" aria-hidden="true" id="configuration"></a><a href="#configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Configuration</h5> |
| <p>Below are the configuration options known to this module:</p> |
| <table> |
| <thead> |
| <tr><th><code>runtime.properties</code></th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.cache.type</code></td><td>Set this to <code>caffeine</code> or leave out parameter</td><td><code>caffeine</code></td></tr> |
| <tr><td><code>druid.cache.sizeInBytes</code></td><td>The maximum size of the cache in bytes on heap. It can be configured as described in <a href="/docs/latest/configuration/human-readable-byte.html">here</a>.</td><td>min(1GiB, Runtime.maxMemory / 10)</td></tr> |
| <tr><td><code>druid.cache.expireAfter</code></td><td>The time (in ms) after an access for which a cache entry may be expired</td><td>None (no time limit)</td></tr> |
| <tr><td><code>druid.cache.cacheExecutorFactory</code></td><td>The executor factory to use for Caffeine maintenance. One of <code>COMMON_FJP</code>, <code>SINGLE_THREAD</code>, or <code>SAME_THREAD</code></td><td>ForkJoinPool common pool (<code>COMMON_FJP</code>)</td></tr> |
| <tr><td><code>druid.cache.evictOnClose</code></td><td>If a close of a namespace (ex: removing a segment from a process) should cause an eager eviction of associated cache values</td><td><code>false</code></td></tr> |
| </tbody> |
| </table> |
| <h5><a class="anchor" aria-hidden="true" id="druidcachecacheexecutorfactory"></a><a href="#druidcachecacheexecutorfactory" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a><code>druid.cache.cacheExecutorFactory</code></h5> |
| <p>Here are the possible values for <code>druid.cache.cacheExecutorFactory</code>, which controls how maintenance tasks are run</p> |
| <ul> |
| <li><code>COMMON_FJP</code> (default) use the common ForkJoinPool. Should use with <a href="https://github.com/apache/druid/pull/4810#issuecomment-329922810">JRE 8u60 or higher</a>. Older versions of the JRE may have worse performance than newer JRE versions.</li> |
| <li><code>SINGLE_THREAD</code> Use a single-threaded executor.</li> |
| <li><code>SAME_THREAD</code> Cache maintenance is done eagerly.</li> |
| </ul> |
| <h5><a class="anchor" aria-hidden="true" id="metrics"></a><a href="#metrics" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Metrics</h5> |
| <p>In addition to the normal cache metrics, the caffeine cache implementation also reports the following in both <code>total</code> and <code>delta</code></p> |
| <table> |
| <thead> |
| <tr><th>Metric</th><th>Description</th><th>Normal value</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>query/cache/caffeine/*/requests</code></td><td>Count of hits or misses</td><td>hit + miss</td></tr> |
| <tr><td><code>query/cache/caffeine/*/loadTime</code></td><td>Length of time caffeine spends loading new values (unused feature)</td><td>0</td></tr> |
| <tr><td><code>query/cache/caffeine/*/evictionBytes</code></td><td>Size in bytes that have been evicted from the cache</td><td>Varies, should tune cache <code>sizeInBytes</code> so that <code>sizeInBytes</code>/<code>evictionBytes</code> is approximately the rate of cache churn you desire</td></tr> |
| </tbody> |
| </table> |
| <h5><a class="anchor" aria-hidden="true" id="memcached"></a><a href="#memcached" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Memcached</h5> |
| <p>Uses memcached as cache backend. This allows all processes to share the same cache.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.cache.expiration</code></td><td>Memcached <a href="https://code.google.com/p/memcached/wiki/NewCommands#Standard_Protocol">expiration time</a>.</td><td>2592000 (30 days)</td></tr> |
| <tr><td><code>druid.cache.timeout</code></td><td>Maximum time in milliseconds to wait for a response from Memcached.</td><td>500</td></tr> |
| <tr><td><code>druid.cache.hosts</code></td><td>Comma separated list of Memcached hosts <code><host:port></code>.</td><td>none</td></tr> |
| <tr><td><code>druid.cache.maxObjectSize</code></td><td>Maximum object size in bytes for a Memcached object.</td><td>52428800 (50 MB)</td></tr> |
| <tr><td><code>druid.cache.memcachedPrefix</code></td><td>Key prefix for all keys in Memcached.</td><td>druid</td></tr> |
| <tr><td><code>druid.cache.numConnections</code></td><td>Number of memcached connections to use.</td><td>1</td></tr> |
| <tr><td><code>druid.cache.protocol</code></td><td>Memcached communication protocol. Can be binary or text.</td><td>binary</td></tr> |
| <tr><td><code>druid.cache.locator</code></td><td>Memcached locator. Can be consistent or array_mod.</td><td>consistent</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="hybrid"></a><a href="#hybrid" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Hybrid</h4> |
| <p>Uses a combination of any two caches as a two-level L1 / L2 cache. |
| This may be used to combine a local in-memory cache with a remote memcached cache.</p> |
| <p>Cache requests will first check L1 cache before checking L2. |
| If there is an L1 miss and L2 hit, it will also populate L1.</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.cache.l1.type</code></td><td>type of cache to use for L1 cache. See <code>druid.cache.type</code> configuration for valid types.</td><td><code>caffeine</code></td></tr> |
| <tr><td><code>druid.cache.l2.type</code></td><td>type of cache to use for L2 cache. See <code>druid.cache.type</code> configuration for valid types.</td><td><code>caffeine</code></td></tr> |
| <tr><td><code>druid.cache.l1.*</code></td><td>Any property valid for the given type of L1 cache can be set using this prefix. For instance, if you are using a <code>caffeine</code> L1 cache, specify <code>druid.cache.l1.sizeInBytes</code> to set its size.</td><td>defaults are the same as for the given cache type.</td></tr> |
| <tr><td><code>druid.cache.l2.*</code></td><td>Prefix for L2 cache settings, see description for L1.</td><td>defaults are the same as for the given cache type.</td></tr> |
| <tr><td><code>druid.cache.useL2</code></td><td>A boolean indicating whether to query L2 cache, if it's a miss in L1. It makes sense to configure this to <code>false</code> on Historical processes, if L2 is a remote cache like <code>memcached</code>, and this cache also used on brokers, because in this case if a query reached Historical it means that a broker didn't find corresponding results in the same remote cache, so a query to the remote cache from Historical is guaranteed to be a miss.</td><td><code>true</code></td></tr> |
| <tr><td><code>druid.cache.populateL2</code></td><td>A boolean indicating whether to put results into L2 cache.</td><td><code>true</code></td></tr> |
| </tbody> |
| </table> |
| <h2><a class="anchor" aria-hidden="true" id="general-query-configuration"></a><a href="#general-query-configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>General query configuration</h2> |
| <p>This section describes configurations that control behavior of Druid's query types, applicable to Broker, Historical, and MiddleManager processes.</p> |
| <h3><a class="anchor" aria-hidden="true" id="overriding-default-query-context-values"></a><a href="#overriding-default-query-context-values" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Overriding default query context values</h3> |
| <p>Any <a href="../querying/query-context.html#general-parameters">Query Context General Parameter</a> default value can be |
| overridden by setting runtime property in the format of <code>druid.query.default.context.{query_context_key}</code>. |
| <code>druid.query.default.context.{query_context_key}</code> runtime property prefix applies to all current and future |
| query context keys, the same as how query context parameter passed with the query works. Note that the runtime property |
| value can be overridden if value for the same key is explicitly specify in the query contexts.</p> |
| <p>The precedence chain for query context values is as follows:</p> |
| <p>hard-coded default value in Druid code <- runtime property not prefixed with <code>druid.query.default.context</code> |
| <- runtime property prefixed with <code>druid.query.default.context</code> <- context parameter in the query</p> |
| <p>Note that not all query context key has a runtime property not prefixed with <code>druid.query.default.context</code> that can |
| override the hard-coded default value. For example, <code>maxQueuedBytes</code> has <code>druid.broker.http.maxQueuedBytes</code> |
| but <code>joinFilterRewriteMaxSize</code> does not. Hence, the only way of overriding <code>joinFilterRewriteMaxSize</code> hard-coded default |
| value is with runtime property <code>druid.query.default.context.joinFilterRewriteMaxSize</code>.</p> |
| <p>To further elaborate on the previous example:</p> |
| <p>If neither <code>druid.broker.http.maxQueuedBytes</code> or <code>druid.query.default.context.maxQueuedBytes</code> is set and |
| the query does not have <code>maxQueuedBytes</code> in the context, then the hard-coded value in Druid code is use. |
| If runtime property only contains <code>druid.broker.http.maxQueuedBytes=x</code> and query does not have <code>maxQueuedBytes</code> in the |
| context, then the value of the property, <code>x</code>, is use. However, if query does have <code>maxQueuedBytes</code> in the context, |
| then that value is use instead. |
| If runtime property only contains <code>druid.query.default.context.maxQueuedBytes=y</code> OR runtime property contains both |
| <code>druid.broker.http.maxQueuedBytes=x</code> and <code>druid.query.default.context.maxQueuedBytes=y</code>, then the value of |
| <code>druid.query.default.context.maxQueuedBytes</code>, <code>y</code>, is use (given that query does not have <code>maxQueuedBytes</code> in the |
| context). If query does have <code>maxQueuedBytes</code> in the context, then that value is use instead.</p> |
| <h3><a class="anchor" aria-hidden="true" id="topn-query-config"></a><a href="#topn-query-config" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>TopN query config</h3> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.query.topN.minTopNThreshold</code></td><td>See <a href="../querying/topnquery.html#aliasing">TopN Aliasing</a> for details.</td><td>1000</td></tr> |
| </tbody> |
| </table> |
| <h3><a class="anchor" aria-hidden="true" id="search-query-config"></a><a href="#search-query-config" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Search query config</h3> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.query.search.maxSearchLimit</code></td><td>Maximum number of search results to return.</td><td>1000</td></tr> |
| <tr><td><code>druid.query.search.searchStrategy</code></td><td>Default search query strategy.</td><td>useIndexes</td></tr> |
| </tbody> |
| </table> |
| <h3><a class="anchor" aria-hidden="true" id="segmentmetadata-query-config"></a><a href="#segmentmetadata-query-config" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>SegmentMetadata query config</h3> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.query.segmentMetadata.defaultHistory</code></td><td>When no interval is specified in the query, use a default interval of defaultHistory before the end time of the most recent segment, specified in ISO8601 format. This property also controls the duration of the default interval used by GET /druid/v2/datasources/{dataSourceName} interactions for retrieving datasource dimensions/metrics.</td><td>P1W</td></tr> |
| <tr><td><code>druid.query.segmentMetadata.defaultAnalysisTypes</code></td><td>This can be used to set the Default Analysis Types for all segment metadata queries, this can be overridden when making the query</td><td>["cardinality", "interval", "minmax"]</td></tr> |
| </tbody> |
| </table> |
| <h3><a class="anchor" aria-hidden="true" id="groupby-query-config"></a><a href="#groupby-query-config" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>GroupBy query config</h3> |
| <p>This section describes the configurations for groupBy queries. You can set the runtime properties in the <code>runtime.properties</code> file on Broker, Historical, and MiddleManager processes. You can set the query context parameters through the <a href="/docs/latest/querying/query-context.html">query context</a>.</p> |
| <h4><a class="anchor" aria-hidden="true" id="configurations-for-groupby-v2"></a><a href="#configurations-for-groupby-v2" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Configurations for groupBy v2</h4> |
| <p>Supported runtime properties:</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.query.groupBy.maxMergingDictionarySize</code></td><td>Maximum amount of heap space (approximately) to use for the string dictionary during merging. When the dictionary exceeds this size, a spill to disk will be triggered.</td><td>100000000</td></tr> |
| <tr><td><code>druid.query.groupBy.maxOnDiskStorage</code></td><td>Maximum amount of disk space to use, per-query, for spilling result sets to disk when either the merging buffer or the dictionary fills up. Queries that exceed this limit will fail. Set to zero to disable disk spilling.</td><td>0 (disabled)</td></tr> |
| </tbody> |
| </table> |
| <p>Supported query contexts:</p> |
| <table> |
| <thead> |
| <tr><th>Key</th><th>Description</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>maxMergingDictionarySize</code></td><td>Can be used to lower the value of <code>druid.query.groupBy.maxMergingDictionarySize</code> for this query.</td></tr> |
| <tr><td><code>maxOnDiskStorage</code></td><td>Can be used to lower the value of <code>druid.query.groupBy.maxOnDiskStorage</code> for this query.</td></tr> |
| </tbody> |
| </table> |
| <h3><a class="anchor" aria-hidden="true" id="advanced-configurations"></a><a href="#advanced-configurations" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Advanced configurations</h3> |
| <h4><a class="anchor" aria-hidden="true" id="common-configurations-for-all-groupby-strategies"></a><a href="#common-configurations-for-all-groupby-strategies" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Common configurations for all groupBy strategies</h4> |
| <p>Supported runtime properties:</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.query.groupBy.defaultStrategy</code></td><td>Default groupBy query strategy.</td><td>v2</td></tr> |
| <tr><td><code>druid.query.groupBy.singleThreaded</code></td><td>Merge results using a single thread.</td><td>false</td></tr> |
| </tbody> |
| </table> |
| <p>Supported query contexts:</p> |
| <table> |
| <thead> |
| <tr><th>Key</th><th>Description</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>groupByStrategy</code></td><td>Overrides the value of <code>druid.query.groupBy.defaultStrategy</code> for this query.</td></tr> |
| <tr><td><code>groupByIsSingleThreaded</code></td><td>Overrides the value of <code>druid.query.groupBy.singleThreaded</code> for this query.</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="groupby-v2-configurations"></a><a href="#groupby-v2-configurations" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>GroupBy v2 configurations</h4> |
| <p>Supported runtime properties:</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.query.groupBy.bufferGrouperInitialBuckets</code></td><td>Initial number of buckets in the off-heap hash table used for grouping results. Set to 0 to use a reasonable default (1024).</td><td>0</td></tr> |
| <tr><td><code>druid.query.groupBy.bufferGrouperMaxLoadFactor</code></td><td>Maximum load factor of the off-heap hash table used for grouping results. When the load factor exceeds this size, the table will be grown or spilled to disk. Set to 0 to use a reasonable default (0.7).</td><td>0</td></tr> |
| <tr><td><code>druid.query.groupBy.forceHashAggregation</code></td><td>Force to use hash-based aggregation.</td><td>false</td></tr> |
| <tr><td><code>druid.query.groupBy.intermediateCombineDegree</code></td><td>Number of intermediate processes combined together in the combining tree. Higher degrees will need less threads which might be helpful to improve the query performance by reducing the overhead of too many threads if the server has sufficiently powerful CPU cores.</td><td>8</td></tr> |
| <tr><td><code>druid.query.groupBy.numParallelCombineThreads</code></td><td>Hint for the number of parallel combining threads. This should be larger than 1 to turn on the parallel combining feature. The actual number of threads used for parallel combining is min(<code>druid.query.groupBy.numParallelCombineThreads</code>, <code>druid.processing.numThreads</code>).</td><td>1 (disabled)</td></tr> |
| </tbody> |
| </table> |
| <p>Supported query contexts:</p> |
| <table> |
| <thead> |
| <tr><th>Key</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>bufferGrouperInitialBuckets</code></td><td>Overrides the value of <code>druid.query.groupBy.bufferGrouperInitialBuckets</code> for this query.</td><td>None</td></tr> |
| <tr><td><code>bufferGrouperMaxLoadFactor</code></td><td>Overrides the value of <code>druid.query.groupBy.bufferGrouperMaxLoadFactor</code> for this query.</td><td>None</td></tr> |
| <tr><td><code>forceHashAggregation</code></td><td>Overrides the value of <code>druid.query.groupBy.forceHashAggregation</code></td><td>None</td></tr> |
| <tr><td><code>intermediateCombineDegree</code></td><td>Overrides the value of <code>druid.query.groupBy.intermediateCombineDegree</code></td><td>None</td></tr> |
| <tr><td><code>numParallelCombineThreads</code></td><td>Overrides the value of <code>druid.query.groupBy.numParallelCombineThreads</code></td><td>None</td></tr> |
| <tr><td><code>sortByDimsFirst</code></td><td>Sort the results first by dimension values and then by timestamp.</td><td>false</td></tr> |
| <tr><td><code>forceLimitPushDown</code></td><td>When all fields in the orderby are part of the grouping key, the broker will push limit application down to the Historical processes. When the sorting order uses fields that are not in the grouping key, applying this optimization can result in approximate results with unknown accuracy, so this optimization is disabled by default in that case. Enabling this context flag turns on limit push down for limit/orderbys that contain non-grouping key columns.</td><td>false</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="groupby-v1-configurations"></a><a href="#groupby-v1-configurations" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>GroupBy v1 configurations</h4> |
| <p>Supported runtime properties:</p> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.query.groupBy.maxIntermediateRows</code></td><td>Maximum number of intermediate rows for the per-segment grouping engine. This is a tuning parameter that does not impose a hard limit; rather, it potentially shifts merging work from the per-segment engine to the overall merging index. Queries that exceed this limit will not fail.</td><td>50000</td></tr> |
| <tr><td><code>druid.query.groupBy.maxResults</code></td><td>Maximum number of results. Queries that exceed this limit will fail.</td><td>500000</td></tr> |
| </tbody> |
| </table> |
| <p>Supported query contexts:</p> |
| <table> |
| <thead> |
| <tr><th>Key</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>maxIntermediateRows</code></td><td>Can be used to lower the value of <code>druid.query.groupBy.maxIntermediateRows</code> for this query.</td><td>None</td></tr> |
| <tr><td><code>maxResults</code></td><td>Can be used to lower the value of <code>druid.query.groupBy.maxResults</code> for this query.</td><td>None</td></tr> |
| <tr><td><code>useOffheap</code></td><td>Set to true to store aggregations off-heap when merging results.</td><td>false</td></tr> |
| </tbody> |
| </table> |
| <h3><a class="anchor" aria-hidden="true" id="router"></a><a href="#router" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Router</h3> |
| <h4><a class="anchor" aria-hidden="true" id="router-process-configs"></a><a href="#router-process-configs" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Router Process Configs</h4> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.host</code></td><td>The host for the current process. This is used to advertise the current processes location as reachable from another process and should generally be specified such that <code>http://${druid.host}/</code> could actually talk to this process</td><td>InetAddress.getLocalHost().getCanonicalHostName()</td></tr> |
| <tr><td><code>druid.bindOnHost</code></td><td>Indicating whether the process's internal jetty server bind on <code>druid.host</code>. Default is false, which means binding to all interfaces.</td><td>false</td></tr> |
| <tr><td><code>druid.plaintextPort</code></td><td>This is the port to actually listen on; unless port mapping is used, this will be the same port as is on <code>druid.host</code></td><td>8888</td></tr> |
| <tr><td><code>druid.tlsPort</code></td><td>TLS port for HTTPS connector, if <a href="/docs/latest/operations/tls-support.html">druid.enableTlsPort</a> is set then this config will be used. If <code>druid.host</code> contains port then that port will be ignored. This should be a non-negative Integer.</td><td>9088</td></tr> |
| <tr><td><code>druid.service</code></td><td>The name of the service. This is used as a dimension when emitting metrics and alerts to differentiate between the various services</td><td>druid/router</td></tr> |
| </tbody> |
| </table> |
| <h4><a class="anchor" aria-hidden="true" id="runtime-configuration"></a><a href="#runtime-configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Runtime Configuration</h4> |
| <table> |
| <thead> |
| <tr><th>Property</th><th>Description</th><th>Default</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>druid.router.defaultBrokerServiceName</code></td><td>The default Broker to connect to in case service discovery fails.</td><td>druid/broker</td></tr> |
| <tr><td><code>druid.router.tierToBrokerMap</code></td><td>Queries for a certain tier of data are routed to their appropriate Broker. This value should be an ordered JSON map of tiers to Broker names. The priority of Brokers is based on the ordering.</td><td>{"_default_tier": "<defaultBrokerServiceName>"}</td></tr> |
| <tr><td><code>druid.router.defaultRule</code></td><td>The default rule for all datasources.</td><td>"_default"</td></tr> |
| <tr><td><code>druid.router.pollPeriod</code></td><td>How often to poll for new rules.</td><td>PT1M</td></tr> |
| <tr><td><code>druid.router.strategies</code></td><td>Please see <a href="../design/router.html#router-strategies">Router Strategies</a> for details.</td><td>[{"type":"timeBoundary"},{"type":"priority"}]</td></tr> |
| <tr><td><code>druid.router.avatica.balancer.type</code></td><td>Class to use for balancing Avatica queries across Brokers. Please see <a href="../design/router.html#avatica-query-balancing">Avatica Query Balancing</a>.</td><td>rendezvousHash</td></tr> |
| <tr><td><code>druid.router.managementProxy.enabled</code></td><td>Enables the Router's <a href="../design/router.html#router-as-management-proxy">management proxy</a> functionality.</td><td>false</td></tr> |
| <tr><td><code>druid.router.http.numConnections</code></td><td>Size of connection pool for the Router to connect to Broker processes. If there are more queries than this number that all need to speak to the same process, then they will queue up.</td><td><code>20</code></td></tr> |
| <tr><td><code>druid.router.http.readTimeout</code></td><td>The timeout for data reads from Broker processes.</td><td><code>PT15M</code></td></tr> |
| <tr><td><code>druid.router.http.numMaxThreads</code></td><td>Maximum number of worker threads to handle HTTP requests and responses</td><td><code>max(10, ((number of cores * 17) / 16 + 2) + 30)</code></td></tr> |
| <tr><td><code>druid.router.http.numRequestsQueued</code></td><td>Maximum number of requests that may be queued to a destination</td><td><code>1024</code></td></tr> |
| <tr><td><code>druid.router.http.requestBuffersize</code></td><td>Size of the content buffer for receiving requests. These buffers are only used for active connections that have requests with bodies that will not fit within the header buffer</td><td><code>8 * 1024</code></td></tr> |
| </tbody> |
| </table> |
| </span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/latest/development/geo.html"><span class="arrow-prev">← </span><span>Spatial filters</span></a><a class="docs-next button" href="/docs/latest/development/extensions.html"><span>Extensions</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#recommended-configuration-file-organization">Recommended Configuration File Organization</a></li><li><a href="#common-configurations">Common Configurations</a><ul class="toc-headings"><li><a href="#jvm-configuration-best-practices">JVM Configuration Best Practices</a></li><li><a href="#extensions">Extensions</a></li><li><a href="#modules">Modules</a></li><li><a href="#zookeeper">Zookeeper</a></li><li><a href="#exhibitor">Exhibitor</a></li><li><a href="#tls">TLS</a></li><li><a href="#authentication-and-authorization">Authentication and Authorization</a></li><li><a href="#startup-logging">Startup Logging</a></li><li><a href="#request-logging">Request Logging</a></li><li><a href="#audit-logging">Audit Logging</a></li><li><a href="#enabling-metrics">Enabling Metrics</a></li><li><a href="#emitting-metrics">Emitting Metrics</a></li><li><a href="#metadata-storage">Metadata storage</a></li><li><a href="#deep-storage">Deep storage</a></li><li><a href="#task-logging">Task Logging</a></li><li><a href="#overlord-discovery">Overlord Discovery</a></li><li><a href="#coordinator-discovery">Coordinator Discovery</a></li><li><a href="#announcing-segments">Announcing Segments</a></li><li><a href="#javascript">JavaScript</a></li><li><a href="#double-column-storage">Double Column storage</a></li><li><a href="#sql-compatible-null-handling">SQL compatible null handling</a></li><li><a href="#http-client">HTTP Client</a></li></ul></li><li><a href="#master-server">Master Server</a><ul class="toc-headings"><li><a href="#coordinator">Coordinator</a></li><li><a href="#overlord">Overlord</a></li></ul></li><li><a href="#data-server">Data Server</a><ul class="toc-headings"><li><a href="#middlemanager-and-peons">MiddleManager and Peons</a></li><li><a href="#indexer">Indexer</a></li><li><a href="#historical">Historical</a></li></ul></li><li><a href="#query-server">Query Server</a><ul class="toc-headings"><li><a href="#broker">Broker</a></li></ul></li><li><a href="#cache-configuration">Cache Configuration</a><ul class="toc-headings"><li><a href="#cache-type">Cache Type</a></li></ul></li><li><a href="#general-query-configuration">General query configuration</a><ul class="toc-headings"><li><a href="#overriding-default-query-context-values">Overriding default query context values</a></li><li><a href="#topn-query-config">TopN query config</a></li><li><a href="#search-query-config">Search query config</a></li><li><a href="#segmentmetadata-query-config">SegmentMetadata query config</a></li><li><a href="#groupby-query-config">GroupBy query config</a></li><li><a href="#advanced-configurations">Advanced configurations</a></li><li><a href="#router">Router</a></li></ul></li></ul></nav></div><footer class="nav-footer druid-footer" id="footer"><div class="container"><div class="text-center"><p><a href="/technology">Technology</a> · <a href="/use-cases">Use Cases</a> · <a href="/druid-powered">Powered by Druid</a> · <a href="/docs/latest/latest">Docs</a> · <a href="/community/">Community</a> · <a href="/downloads.html">Download</a> · <a href="/faq">FAQ</a></p></div><div class="text-center"><a title="Join the user group" href="https://groups.google.com/forum/#!forum/druid-user" target="_blank"><span class="fa fa-comments"></span></a> · <a title="Follow Druid" href="https://twitter.com/druidio" target="_blank"><span class="fab fa-twitter"></span></a> · <a title="Download via Apache" href="https://www.apache.org/dyn/closer.cgi?path=/incubator/druid/{{ site.druid_versions[0].versions[0].version }}/apache-druid-{{ site.druid_versions[0].versions[0].version }}-bin.tar.gz" target="_blank"><span class="fas fa-feather"></span></a> · <a title="GitHub" href="https://github.com/apache/druid" target="_blank"><span class="fab fa-github"></span></a></div><div class="text-center license">Copyright © 2019 <a href="https://www.apache.org/" target="_blank">Apache Software Foundation</a>.<br/>Except where otherwise noted, licensed under <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">CC BY-SA 4.0</a>.<br/>Apache Druid, Druid, and the Druid logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</div></div></footer></div><script type="text/javascript" src="https://cdn.jsdelivr.net/docsearch.js/1/docsearch.min.js"></script><script> |
| document.addEventListener('keyup', function(e) { |
| if (e.target !== document.body) { |
| return; |
| } |
| // keyCode for '/' (slash) |
| if (e.keyCode === 191) { |
| const search = document.getElementById('search_input_react'); |
| search && search.focus(); |
| } |
| }); |
| </script><script> |
| var search = docsearch({ |
| |
| apiKey: '2de99082a9f38e49dfaa059bbe4c901d', |
| indexName: 'apache_druid', |
| inputSelector: '#search_input_react', |
| algoliaOptions: {"facetFilters":["language:en","version:0.20.0"]} |
| }); |
| </script></body></html> |
