| "use strict";(self.webpackChunk=self.webpackChunk||[]).push([[6785],{15680:(e,n,t)=>{t.d(n,{xA:()=>g,yg:()=>m});var a=t(96540);function r(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function i(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);n&&(a=a.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,a)}return t}function o(e){for(var n=1;n<arguments.length;n++){var t=null!=arguments[n]?arguments[n]:{};n%2?i(Object(t),!0).forEach((function(n){r(e,n,t[n])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(t)):i(Object(t)).forEach((function(n){Object.defineProperty(e,n,Object.getOwnPropertyDescriptor(t,n))}))}return e}function l(e,n){if(null==e)return{};var t,a,r=function(e,n){if(null==e)return{};var t,a,r={},i=Object.keys(e);for(a=0;a<i.length;a++)t=i[a],n.indexOf(t)>=0||(r[t]=e[t]);return r}(e,n);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(a=0;a<i.length;a++)t=i[a],n.indexOf(t)>=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(r[t]=e[t])}return r}var s=a.createContext({}),u=function(e){var n=a.useContext(s),t=n;return e&&(t="function"==typeof e?e(n):o(o({},n),e)),t},g=function(e){var n=u(e.components);return a.createElement(s.Provider,{value:n},e.children)},d="mdxType",p={inlineCode:"code",wrapper:function(e){var n=e.children;return a.createElement(a.Fragment,{},n)}},y=a.forwardRef((function(e,n){var t=e.components,r=e.mdxType,i=e.originalType,s=e.parentName,g=l(e,["components","mdxType","originalType","parentName"]),d=u(t),y=r,m=d["".concat(s,".").concat(y)]||d[y]||p[y]||i;return t?a.createElement(m,o(o({ref:n},g),{},{components:t})):a.createElement(m,o({ref:n},g))}));function m(e,n){var t=arguments,r=n&&n.mdxType;if("string"==typeof e||r){var i=t.length,o=new Array(i);o[0]=y;var l={};for(var s in n)hasOwnProperty.call(n,s)&&(l[s]=n[s]);l.originalType=e,l[d]="string"==typeof e?e:r,o[1]=l;for(var u=2;u<i;u++)o[u]=t[u];return a.createElement.apply(null,o)}return a.createElement.apply(null,t)}y.displayName="MDXCreateElement"},98585:(e,n,t)=>{t.r(n),t.d(n,{assets:()=>g,contentTitle:()=>s,default:()=>m,frontMatter:()=>l,metadata:()=>u,toc:()=>d});var a=t(58168),r=t(98587),i=(t(96540),t(15680)),o=["components"],l={id:"groupbyquery",title:"GroupBy queries",sidebar_label:"GroupBy"},s=void 0,u={unversionedId:"querying/groupbyquery",id:"querying/groupbyquery",title:"GroupBy queries",description:"\x3c!--",source:"@site/docs/29.0.0/querying/groupbyquery.md",sourceDirName:"querying",slug:"/querying/groupbyquery",permalink:"/docs/29.0.0/querying/groupbyquery",draft:!1,tags:[],version:"current",frontMatter:{id:"groupbyquery",title:"GroupBy queries",sidebar_label:"GroupBy"},sidebar:"docs",previous:{title:"TopN",permalink:"/docs/29.0.0/querying/topnquery"},next:{title:"Scan",permalink:"/docs/29.0.0/querying/scan-query"}},g={},d=[{value:"Behavior on multi-value dimensions",id:"behavior-on-multi-value-dimensions",level:2},{value:"More on subtotalsSpec",id:"more-on-subtotalsspec",level:2},{value:"Implementation details",id:"implementation-details",level:2},{value:"Memory tuning and resource limits",id:"memory-tuning-and-resource-limits",level:3},{value:"Performance tuning for groupBy",id:"performance-tuning-for-groupby",level:3},{value:"Limit pushdown optimization",id:"limit-pushdown-optimization",level:4},{value:"Optimizing hash table",id:"optimizing-hash-table",level:4},{value:"Parallel combine",id:"parallel-combine",level:4},{value:"Alternatives",id:"alternatives",level:3},{value:"Nested groupBys",id:"nested-groupbys",level:3},{value:"Configurations",id:"configurations",level:3},{value:"Advanced configurations",id:"advanced-configurations",level:3},{value:"Array based result rows",id:"array-based-result-rows",level:4}],p={toc:d},y="wrapper";function m(e){var n=e.components,t=(0,r.A)(e,o);return(0,i.yg)(y,(0,a.A)({},p,t,{components:n,mdxType:"MDXLayout"}),(0,i.yg)("admonition",{type:"info"},(0,i.yg)("p",{parentName:"admonition"}," Apache Druid supports two query languages: ",(0,i.yg)("a",{parentName:"p",href:"/docs/29.0.0/querying/sql"},"Druid SQL")," and ",(0,i.yg)("a",{parentName:"p",href:"/docs/29.0.0/querying/"},"native queries"),".\nThis document describes a query\ntype in the native language. For information about when Druid SQL will use this query type, refer to the\n",(0,i.yg)("a",{parentName:"p",href:"/docs/29.0.0/querying/sql-translation#query-types"},"SQL documentation"),".")),(0,i.yg)("p",null,"These types of Apache Druid queries take a groupBy query object and return an array of JSON objects where each object represents a\ngrouping asked for by the query."),(0,i.yg)("admonition",{type:"info"},(0,i.yg)("p",{parentName:"admonition"}," Note: If you are doing aggregations with time as your only grouping, or an ordered groupBy over a single dimension,\nconsider ",(0,i.yg)("a",{parentName:"p",href:"/docs/29.0.0/querying/timeseriesquery"},"Timeseries")," and ",(0,i.yg)("a",{parentName:"p",href:"/docs/29.0.0/querying/topnquery"},"TopN")," queries as well as\ngroupBy. Their performance may be better in some cases. See ",(0,i.yg)("a",{parentName:"p",href:"#alternatives"},"Alternatives")," below for more details.")),(0,i.yg)("p",null,"An example groupBy query object is shown below:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'{\n "queryType": "groupBy",\n "dataSource": "sample_datasource",\n "granularity": "day",\n "dimensions": ["country", "device"],\n "limitSpec": { "type": "default", "limit": 5000, "columns": ["country", "data_transfer"] },\n "filter": {\n "type": "and",\n "fields": [\n { "type": "selector", "dimension": "carrier", "value": "AT&T" },\n { "type": "or",\n "fields": [\n { "type": "selector", "dimension": "make", "value": "Apple" },\n { "type": "selector", "dimension": "make", "value": "Samsung" }\n ]\n }\n ]\n },\n "aggregations": [\n { "type": "longSum", "name": "total_usage", "fieldName": "user_count" },\n { "type": "doubleSum", "name": "data_transfer", "fieldName": "data_transfer" }\n ],\n "postAggregations": [\n { "type": "arithmetic",\n "name": "avg_usage",\n "fn": "/",\n "fields": [\n { "type": "fieldAccess", "fieldName": "data_transfer" },\n { "type": "fieldAccess", "fieldName": "total_usage" }\n ]\n }\n ],\n "intervals": [ "2012-01-01T00:00:00.000/2012-01-03T00:00:00.000" ],\n "having": {\n "type": "greaterThan",\n "aggregation": "total_usage",\n "value": 100\n }\n}\n')),(0,i.yg)("p",null,"Following are main parts to a groupBy query:"),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"property"),(0,i.yg)("th",{parentName:"tr",align:null},"description"),(0,i.yg)("th",{parentName:"tr",align:null},"required?"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"queryType"),(0,i.yg)("td",{parentName:"tr",align:null},'This String should always be "groupBy"; this is the first thing Druid looks at to figure out how to interpret the query'),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"dataSource"),(0,i.yg)("td",{parentName:"tr",align:null},"A String or Object defining the data source to query, very similar to a table in a relational database. See ",(0,i.yg)("a",{parentName:"td",href:"/docs/29.0.0/querying/datasource"},"DataSource")," for more information."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"dimensions"),(0,i.yg)("td",{parentName:"tr",align:null},"A JSON list of dimensions to do the groupBy over; or see ",(0,i.yg)("a",{parentName:"td",href:"/docs/29.0.0/querying/dimensionspecs"},"DimensionSpec")," for ways to extract dimensions."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"virtualColumns"),(0,i.yg)("td",{parentName:"tr",align:null},"A JSON list of ",(0,i.yg)("a",{parentName:"td",href:"/docs/29.0.0/querying/virtual-columns"},"virtual columns"),". You can reference the virtual columns in ",(0,i.yg)("inlineCode",{parentName:"td"},"dimensions"),", ",(0,i.yg)("inlineCode",{parentName:"td"},"aggregations"),", or ",(0,i.yg)("inlineCode",{parentName:"td"},"postAggregations"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"no (default none)")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"limitSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"See ",(0,i.yg)("a",{parentName:"td",href:"/docs/29.0.0/querying/limitspec"},"LimitSpec"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"no")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"having"),(0,i.yg)("td",{parentName:"tr",align:null},"See ",(0,i.yg)("a",{parentName:"td",href:"/docs/29.0.0/querying/having"},"Having"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"no")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"granularity"),(0,i.yg)("td",{parentName:"tr",align:null},"Defines the granularity of the query. See ",(0,i.yg)("a",{parentName:"td",href:"/docs/29.0.0/querying/granularities"},"Granularities")),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"filter"),(0,i.yg)("td",{parentName:"tr",align:null},"See ",(0,i.yg)("a",{parentName:"td",href:"/docs/29.0.0/querying/filters"},"Filters")),(0,i.yg)("td",{parentName:"tr",align:null},"no")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"aggregations"),(0,i.yg)("td",{parentName:"tr",align:null},"See ",(0,i.yg)("a",{parentName:"td",href:"/docs/29.0.0/querying/aggregations"},"Aggregations")),(0,i.yg)("td",{parentName:"tr",align:null},"no")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"postAggregations"),(0,i.yg)("td",{parentName:"tr",align:null},"See ",(0,i.yg)("a",{parentName:"td",href:"/docs/29.0.0/querying/post-aggregations"},"Post Aggregations")),(0,i.yg)("td",{parentName:"tr",align:null},"no")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"intervals"),(0,i.yg)("td",{parentName:"tr",align:null},"A JSON Object representing ISO-8601 Intervals. This defines the time ranges to run the query over."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"subtotalsSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"A JSON array of arrays to return additional result sets for groupings of subsets of top level ",(0,i.yg)("inlineCode",{parentName:"td"},"dimensions"),". It is ",(0,i.yg)("a",{parentName:"td",href:"/docs/29.0.0/querying/groupbyquery#more-on-subtotalsspec"},"described later")," in more detail."),(0,i.yg)("td",{parentName:"tr",align:null},"no")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"context"),(0,i.yg)("td",{parentName:"tr",align:null},"An additional JSON Object which can be used to specify certain flags."),(0,i.yg)("td",{parentName:"tr",align:null},"no")))),(0,i.yg)("p",null,"To pull it all together, the above query would return ",(0,i.yg)("em",{parentName:"p"},"n","*","m")," data points, up to a maximum of 5000 points, where n is the cardinality of the ",(0,i.yg)("inlineCode",{parentName:"p"},"country")," dimension, m is the cardinality of the ",(0,i.yg)("inlineCode",{parentName:"p"},"device")," dimension, each day between 2012-01-01 and 2012-01-03, from the ",(0,i.yg)("inlineCode",{parentName:"p"},"sample_datasource")," table. Each data point contains the (long) sum of ",(0,i.yg)("inlineCode",{parentName:"p"},"total_usage")," if the value of the data point is greater than 100, the (double) sum of ",(0,i.yg)("inlineCode",{parentName:"p"},"data_transfer")," and the (double) result of ",(0,i.yg)("inlineCode",{parentName:"p"},"total_usage")," divided by ",(0,i.yg)("inlineCode",{parentName:"p"},"data_transfer")," for the filter set for a particular grouping of ",(0,i.yg)("inlineCode",{parentName:"p"},"country")," and ",(0,i.yg)("inlineCode",{parentName:"p"},"device"),". The output looks like this:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'[\n {\n "version" : "v1",\n "timestamp" : "2012-01-01T00:00:00.000Z",\n "event" : {\n "country" : <some_dim_value_one>,\n "device" : <some_dim_value_two>,\n "total_usage" : <some_value_one>,\n "data_transfer" :<some_value_two>,\n "avg_usage" : <some_avg_usage_value>\n }\n },\n {\n "version" : "v1",\n "timestamp" : "2012-01-01T00:00:12.000Z",\n "event" : {\n "dim1" : <some_other_dim_value_one>,\n "dim2" : <some_other_dim_value_two>,\n "sample_name1" : <some_other_value_one>,\n "sample_name2" :<some_other_value_two>,\n "avg_usage" : <some_other_avg_usage_value>\n }\n },\n...\n]\n')),(0,i.yg)("h2",{id:"behavior-on-multi-value-dimensions"},"Behavior on multi-value dimensions"),(0,i.yg)("p",null,"groupBy queries can group on multi-value dimensions. When grouping on a multi-value dimension, ",(0,i.yg)("em",{parentName:"p"},"all")," values\nfrom matching rows will be used to generate one group per value. It's possible for a query to return more groups than\nthere are rows. For example, a groupBy on the dimension ",(0,i.yg)("inlineCode",{parentName:"p"},"tags")," with filter ",(0,i.yg)("inlineCode",{parentName:"p"},'"t1" AND "t3"')," would match only row1, and\ngenerate a result with three groups: ",(0,i.yg)("inlineCode",{parentName:"p"},"t1"),", ",(0,i.yg)("inlineCode",{parentName:"p"},"t2"),", and ",(0,i.yg)("inlineCode",{parentName:"p"},"t3"),". If you only need to include values that match\nyour filter, you can use a ",(0,i.yg)("a",{parentName:"p",href:"/docs/29.0.0/querying/dimensionspecs#filtered-dimensionspecs"},"filtered dimensionSpec"),". This can also\nimprove performance."),(0,i.yg)("p",null,"See ",(0,i.yg)("a",{parentName:"p",href:"/docs/29.0.0/querying/multi-value-dimensions"},"Multi-value dimensions")," for more details."),(0,i.yg)("h2",{id:"more-on-subtotalsspec"},"More on subtotalsSpec"),(0,i.yg)("p",null,'The subtotals feature allows computation of multiple sub-groupings in a single query. To use this feature, add a "subtotalsSpec" to your query as a list of subgroup dimension sets. It should contain the ',(0,i.yg)("inlineCode",{parentName:"p"},"outputName")," from dimensions in your ",(0,i.yg)("inlineCode",{parentName:"p"},"dimensions")," attribute, in the same order as they appear in the ",(0,i.yg)("inlineCode",{parentName:"p"},"dimensions")," attribute (although, of course, you may skip some). "),(0,i.yg)("p",null,"For example, consider a groupBy query like this one:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'{\n"type": "groupBy",\n ...\n ...\n"dimensions": [\n {\n "type" : "default",\n "dimension" : "d1col",\n "outputName": "D1"\n },\n {\n "type" : "extraction",\n "dimension" : "d2col",\n "outputName" : "D2",\n "extractionFn" : extraction_func\n },\n {\n "type":"lookup",\n "dimension":"d3col",\n "outputName":"D3",\n "name":"my_lookup"\n }\n],\n...\n...\n"subtotalsSpec":[ ["D1", "D2", D3"], ["D1", "D3"], ["D3"]],\n..\n\n}\n')),(0,i.yg)("p",null,'The result of the subtotalsSpec would be equivalent to concatenating the result of three groupBy queries, with the "dimensions" field being ',(0,i.yg)("inlineCode",{parentName:"p"},'["D1", "D2", D3"]'),", ",(0,i.yg)("inlineCode",{parentName:"p"},'["D1", "D3"]')," and ",(0,i.yg)("inlineCode",{parentName:"p"},'["D3"]'),", given the ",(0,i.yg)("inlineCode",{parentName:"p"},"DimensionSpec")," shown above.\nThe response for the query above would look something like: "),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'[\n {\n "version" : "v1",\n "timestamp" : "t1",\n "event" : { "D1": "..", "D2": "..", "D3": ".." }\n }\n },\n {\n "version" : "v1",\n "timestamp" : "t2",\n "event" : { "D1": "..", "D2": "..", "D3": ".." }\n }\n },\n ...\n ...\n\n {\n "version" : "v1",\n "timestamp" : "t1",\n "event" : { "D1": "..", "D2": null, "D3": ".." }\n }\n },\n {\n "version" : "v1",\n "timestamp" : "t2",\n "event" : { "D1": "..", "D2": null, "D3": ".." }\n }\n },\n ...\n ...\n\n {\n "version" : "v1",\n "timestamp" : "t1",\n "event" : { "D1": null, "D2": null, "D3": ".." }\n }\n },\n {\n "version" : "v1",\n "timestamp" : "t2",\n "event" : { "D1": null, "D2": null, "D3": ".." }\n }\n },\n...\n]\n')),(0,i.yg)("admonition",{type:"info"},(0,i.yg)("p",{parentName:"admonition"}," Notice that dimensions that are not included in an individual subtotalsSpec grouping are returned with a ",(0,i.yg)("inlineCode",{parentName:"p"},"null")," value. This response format represents a behavior change as of Apache Druid 0.18.0.\nIn release 0.17.0 and earlier, such dimensions were entirely excluded from the result. If you were relying on this old behavior to determine whether a particular dimension was not part of\na subtotal grouping, you can now use ",(0,i.yg)("a",{parentName:"p",href:"/docs/29.0.0/querying/aggregations#grouping-aggregator"},"Grouping aggregator")," instead.")),(0,i.yg)("h2",{id:"implementation-details"},"Implementation details"),(0,i.yg)("h3",{id:"memory-tuning-and-resource-limits"},"Memory tuning and resource limits"),(0,i.yg)("p",null,"When using groupBy, four parameters control resource usage and limits:"),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"druid.processing.buffer.sizeBytes"),": size of the off-heap hash table used for aggregation, per query, in bytes. At\nmost ",(0,i.yg)("inlineCode",{parentName:"li"},"druid.processing.numMergeBuffers")," of these will be created at once, which also serves as an upper limit on the\nnumber of concurrently running groupBy queries."),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"druid.query.groupBy.maxSelectorDictionarySize"),": size of the on-heap segment-level dictionary used when grouping on\nstring or array-valued expressions that do not have pre-existing dictionaries. There is at most one dictionary per\nprocessing thread; therefore there are up to ",(0,i.yg)("inlineCode",{parentName:"li"},"druid.processing.numThreads")," of these. Note that the size is based on a\nrough estimate of the dictionary footprint."),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"druid.query.groupBy.maxMergingDictionarySize"),": size of the on-heap query-level dictionary used when grouping on\nany string expression. There is at most one dictionary per concurrently-running query; therefore there are up to\n",(0,i.yg)("inlineCode",{parentName:"li"},"druid.server.http.numThreads")," of these. Note that the size is based on a rough estimate of the dictionary footprint."),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"druid.query.groupBy.maxOnDiskStorage"),": amount of space on disk used for aggregation, per query, in bytes. By default,\nthis is 0, which means aggregation will not use disk.")),(0,i.yg)("p",null,"If ",(0,i.yg)("inlineCode",{parentName:"p"},"maxOnDiskStorage"),' is 0 (the default) then a query that exceeds either the on-heap dictionary limit, or the off-heap\naggregation table limit, will fail with a "Resource limit exceeded" error describing the limit that was exceeded.'),(0,i.yg)("p",null,"If ",(0,i.yg)("inlineCode",{parentName:"p"},"maxOnDiskStorage")," is greater than 0, queries that exceed the in-memory limits will start using disk for aggregation.\nIn this case, when either the on-heap dictionary or off-heap hash table fills up, partially aggregated records will be\nsorted and flushed to disk. Then, both in-memory structures will be cleared out for further aggregation. Queries that\nthen go on to exceed ",(0,i.yg)("inlineCode",{parentName:"p"},"maxOnDiskStorage"),' will fail with a "Resource limit exceeded" error indicating that they ran out of\ndisk space.'),(0,i.yg)("p",null,"With groupBy, cluster operators should make sure that the off-heap hash tables and on-heap merging dictionaries\nwill not exceed available memory for the maximum possible concurrent query load (given by\n",(0,i.yg)("inlineCode",{parentName:"p"},"druid.processing.numMergeBuffers"),"). See the ",(0,i.yg)("a",{parentName:"p",href:"/docs/29.0.0/operations/basic-cluster-tuning"},"basic cluster tuning guide"),"\nfor more details about direct memory usage, organized by Druid process type."),(0,i.yg)("p",null,"Brokers do not need merge buffers for basic groupBy queries. Queries with subqueries (using a ",(0,i.yg)("inlineCode",{parentName:"p"},"query")," dataSource) require one merge buffer if there is a single subquery, or two merge buffers if there is more than one layer of nested subqueries. Queries with ",(0,i.yg)("a",{parentName:"p",href:"/docs/29.0.0/querying/groupbyquery#more-on-subtotalsspec"},"subtotals")," need one merge buffer. These can stack on top of each other: a groupBy query with multiple layers of nested subqueries, and that also uses subtotals, will need three merge buffers."),(0,i.yg)("p",null,"Historicals and ingestion tasks need one merge buffer for each groupBy query, unless ",(0,i.yg)("a",{parentName:"p",href:"/docs/29.0.0/querying/groupbyquery#parallel-combine"},"parallel combination")," is enabled, in which case they need two merge buffers per query."),(0,i.yg)("h3",{id:"performance-tuning-for-groupby"},"Performance tuning for groupBy"),(0,i.yg)("h4",{id:"limit-pushdown-optimization"},"Limit pushdown optimization"),(0,i.yg)("p",null,"Druid pushes down the ",(0,i.yg)("inlineCode",{parentName:"p"},"limit")," spec in groupBy queries to the segments on Historicals wherever possible to early prune unnecessary intermediate results and minimize the amount of data transferred to Brokers. By default, this technique is applied only when all fields in the ",(0,i.yg)("inlineCode",{parentName:"p"},"orderBy")," spec is a subset of the grouping keys. This is because the ",(0,i.yg)("inlineCode",{parentName:"p"},"limitPushDown")," doesn't guarantee the exact results if the ",(0,i.yg)("inlineCode",{parentName:"p"},"orderBy")," spec includes any fields that are not in the grouping keys. However, you can enable this technique even in such cases if you can sacrifice some accuracy for fast query processing like in topN queries. See ",(0,i.yg)("inlineCode",{parentName:"p"},"forceLimitPushDown")," in ",(0,i.yg)("a",{parentName:"p",href:"#advanced-configurations"},"advanced configurations"),"."),(0,i.yg)("h4",{id:"optimizing-hash-table"},"Optimizing hash table"),(0,i.yg)("p",null,"The groupBy engine uses an open addressing hash table for aggregation. The hash table is initialized with a given initial bucket number and gradually grows on buffer full. On hash collisions, the linear probing technique is used."),(0,i.yg)("p",null,"The default number of initial buckets is 1024 and the default max load factor of the hash table is 0.7. If you can see too many collisions in the hash table, you can adjust these numbers. See ",(0,i.yg)("inlineCode",{parentName:"p"},"bufferGrouperInitialBuckets")," and ",(0,i.yg)("inlineCode",{parentName:"p"},"bufferGrouperMaxLoadFactor")," in ",(0,i.yg)("a",{parentName:"p",href:"#advanced-configurations"},"advanced configurations"),"."),(0,i.yg)("h4",{id:"parallel-combine"},"Parallel combine"),(0,i.yg)("p",null,"Once a Historical finishes aggregation using the hash table, it sorts the aggregated results and merges them before sending to the\nBroker for N-way merge aggregation in the broker. By default, Historicals use all their available processing threads\n(configured by ",(0,i.yg)("inlineCode",{parentName:"p"},"druid.processing.numThreads"),") for aggregation, but use a single thread for sorting and merging\naggregates which is an http thread to send data to Brokers."),(0,i.yg)("p",null,"This is to prevent some heavy groupBy queries from blocking other queries. In Druid, the processing threads are shared\nbetween all submitted queries and they are ",(0,i.yg)("em",{parentName:"p"},"not interruptible"),". It means, if a heavy query takes all available\nprocessing threads, all other queries might be blocked until the heavy query is finished. GroupBy queries usually take\nlonger time than timeseries or topN queries, they should release processing threads as soon as possible."),(0,i.yg)("p",null,"However, you might care about the performance of some really heavy groupBy queries. Usually, the performance bottleneck\nof heavy groupBy queries is merging sorted aggregates. In such cases, you can use processing threads for it as well.\nThis is called ",(0,i.yg)("em",{parentName:"p"},"parallel combine"),". To enable parallel combine, see ",(0,i.yg)("inlineCode",{parentName:"p"},"numParallelCombineThreads")," in\n",(0,i.yg)("a",{parentName:"p",href:"#advanced-configurations"},"advanced configurations"),". Note that parallel combine can be enabled only when\ndata is actually spilled (see ",(0,i.yg)("a",{parentName:"p",href:"#memory-tuning-and-resource-limits"},"Memory tuning and resource limits"),")."),(0,i.yg)("p",null,"Once parallel combine is enabled, the groupBy engine can create a combining tree for merging sorted aggregates. Each\nintermediate node of the tree is a thread merging aggregates from the child nodes. The leaf node threads read and merge\naggregates from hash tables including spilled ones. Usually, leaf processes are slower than intermediate nodes because they\nneed to read data from disk. As a result, less threads are used for intermediate nodes by default. You can change the\ndegree of intermediate nodes. See ",(0,i.yg)("inlineCode",{parentName:"p"},"intermediateCombineDegree")," in ",(0,i.yg)("a",{parentName:"p",href:"#advanced-configurations"},"advanced configurations"),"."),(0,i.yg)("p",null,"Please note that each Historical needs two merge buffers to process a groupBy query with parallel combine: one for\ncomputing intermediate aggregates from each segment and another for combining intermediate aggregates in parallel."),(0,i.yg)("h3",{id:"alternatives"},"Alternatives"),(0,i.yg)("p",null,"There are some situations where other query types may be a better choice than groupBy."),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("p",{parentName:"li"},'For queries with no "dimensions" (i.e. grouping by time only) the ',(0,i.yg)("a",{parentName:"p",href:"/docs/29.0.0/querying/timeseriesquery"},"Timeseries query")," will\ngenerally be faster\xa0than groupBy. The major differences are that it is implemented in a fully streaming manner (taking\nadvantage of the fact that segments are already sorted on time) and does not need to use a hash table for merging.")),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("p",{parentName:"li"},'For queries with a single "dimensions" element (i.e. grouping by one string dimension), the ',(0,i.yg)("a",{parentName:"p",href:"/docs/29.0.0/querying/topnquery"},"TopN query"),"\nwill sometimes be faster than groupBy. This is especially true if you are ordering by a metric and find approximate\nresults acceptable."))),(0,i.yg)("h3",{id:"nested-groupbys"},"Nested groupBys"),(0,i.yg)("p",null,'Nested groupBys (dataSource of type "query") are performed with the Broker first running the inner groupBy query in the\nusual way. Next, the outer query is run on the inner query\'s results stream with off-heap fact map and on-heap string\ndictionary that can spill to disk. The outer query is run on the Broker in a single-threaded fashion.'),(0,i.yg)("h3",{id:"configurations"},"Configurations"),(0,i.yg)("p",null,"This section describes the configurations for groupBy queries. You can set the runtime properties in the ",(0,i.yg)("inlineCode",{parentName:"p"},"runtime.properties")," file on Broker, Historical, and MiddleManager processes. You can set the query context parameters through the ",(0,i.yg)("a",{parentName:"p",href:"/docs/29.0.0/querying/query-context"},"query context"),"."),(0,i.yg)("p",null,"Supported runtime properties:"),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Property"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Default"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"druid.query.groupBy.maxSelectorDictionarySize")),(0,i.yg)("td",{parentName:"tr",align:null},"Maximum amount of heap space (approximately) to use for per-segment string dictionaries. If set to ",(0,i.yg)("inlineCode",{parentName:"td"},"0")," (automatic), each query's dictionary can use 10% of the Java heap divided by ",(0,i.yg)("inlineCode",{parentName:"td"},"druid.processing.numMergeBuffers"),", or 1GB, whichever is smaller.",(0,i.yg)("br",null),(0,i.yg)("br",null),"See ",(0,i.yg)("a",{parentName:"td",href:"#memory-tuning-and-resource-limits"},"Memory tuning and resource limits")," for details on changing this property."),(0,i.yg)("td",{parentName:"tr",align:null},"0 (automatic)")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"druid.query.groupBy.maxMergingDictionarySize")),(0,i.yg)("td",{parentName:"tr",align:null},"Maximum amount of heap space (approximately) to use for per-query string dictionaries. When the dictionary exceeds this size, a spill to disk will be triggered. If set to ",(0,i.yg)("inlineCode",{parentName:"td"},"0")," (automatic), each query's dictionary uses 30% of the Java heap divided by ",(0,i.yg)("inlineCode",{parentName:"td"},"druid.processing.numMergeBuffers"),", or 1GB, whichever is smaller.",(0,i.yg)("br",null),(0,i.yg)("br",null),"See ",(0,i.yg)("a",{parentName:"td",href:"#memory-tuning-and-resource-limits"},"Memory tuning and resource limits")," for details on changing this property."),(0,i.yg)("td",{parentName:"tr",align:null},"0 (automatic)")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"druid.query.groupBy.maxOnDiskStorage")),(0,i.yg)("td",{parentName:"tr",align:null},"Maximum amount of disk space to use, per-query, for spilling result sets to disk when either the merging buffer or the dictionary fills up. Queries that exceed this limit will fail. Set to zero to disable disk spilling."),(0,i.yg)("td",{parentName:"tr",align:null},"0 (disabled)")))),(0,i.yg)("p",null,"Supported query contexts:"),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Key"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"maxOnDiskStorage")),(0,i.yg)("td",{parentName:"tr",align:null},"Can be used to lower the value of ",(0,i.yg)("inlineCode",{parentName:"td"},"druid.query.groupBy.maxOnDiskStorage")," for this query.")))),(0,i.yg)("h3",{id:"advanced-configurations"},"Advanced configurations"),(0,i.yg)("p",null,"Supported runtime properties:"),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Property"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Default"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"druid.query.groupBy.singleThreaded")),(0,i.yg)("td",{parentName:"tr",align:null},"Merge results using a single thread."),(0,i.yg)("td",{parentName:"tr",align:null},"false")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"druid.query.groupBy.intermediateResultAsMapCompat")),(0,i.yg)("td",{parentName:"tr",align:null},"Whether Brokers are able to understand map-based result rows. Setting this to ",(0,i.yg)("inlineCode",{parentName:"td"},"true")," adds some overhead to all groupBy queries. It is required for compatibility with data servers running versions older than 0.16.0, which introduced ",(0,i.yg)("a",{parentName:"td",href:"#array-based-result-rows"},"array-based result rows"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"false")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"druid.query.groupBy.bufferGrouperInitialBuckets")),(0,i.yg)("td",{parentName:"tr",align:null},"Initial number of buckets in the off-heap hash table used for grouping results. Set to 0 to use a reasonable default (1024)."),(0,i.yg)("td",{parentName:"tr",align:null},"0")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"druid.query.groupBy.bufferGrouperMaxLoadFactor")),(0,i.yg)("td",{parentName:"tr",align:null},"Maximum load factor of the off-heap hash table used for grouping results. When the load factor exceeds this size, the table will be grown or spilled to disk. Set to 0 to use a reasonable default (0.7)."),(0,i.yg)("td",{parentName:"tr",align:null},"0")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"druid.query.groupBy.forceHashAggregation")),(0,i.yg)("td",{parentName:"tr",align:null},"Force to use hash-based aggregation."),(0,i.yg)("td",{parentName:"tr",align:null},"false")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"druid.query.groupBy.intermediateCombineDegree")),(0,i.yg)("td",{parentName:"tr",align:null},"Number of intermediate nodes combined together in the combining tree. Higher degrees will need less threads which might be helpful to improve the query performance by reducing the overhead of too many threads if the server has sufficiently powerful cpu cores."),(0,i.yg)("td",{parentName:"tr",align:null},"8")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"druid.query.groupBy.numParallelCombineThreads")),(0,i.yg)("td",{parentName:"tr",align:null},"Hint for the number of parallel combining threads. This should be larger than 1 to turn on the parallel combining feature. The actual number of threads used for parallel combining is min(",(0,i.yg)("inlineCode",{parentName:"td"},"druid.query.groupBy.numParallelCombineThreads"),", ",(0,i.yg)("inlineCode",{parentName:"td"},"druid.processing.numThreads"),")."),(0,i.yg)("td",{parentName:"tr",align:null},"1 (disabled)")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"druid.query.groupBy.applyLimitPushDownToSegment")),(0,i.yg)("td",{parentName:"tr",align:null},"If Broker pushes limit down to queryable data server (historicals, peons) then limit results during segment scan. If typically there are a large number of segments taking part in a query on a data server, this setting may counterintuitively reduce performance if enabled."),(0,i.yg)("td",{parentName:"tr",align:null},"false (disabled)")))),(0,i.yg)("p",null,"Supported query contexts:"),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Key"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Default"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"groupByIsSingleThreaded")),(0,i.yg)("td",{parentName:"tr",align:null},"Overrides the value of ",(0,i.yg)("inlineCode",{parentName:"td"},"druid.query.groupBy.singleThreaded")," for this query."),(0,i.yg)("td",{parentName:"tr",align:null},"None")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"bufferGrouperInitialBuckets")),(0,i.yg)("td",{parentName:"tr",align:null},"Overrides the value of ",(0,i.yg)("inlineCode",{parentName:"td"},"druid.query.groupBy.bufferGrouperInitialBuckets")," for this query."),(0,i.yg)("td",{parentName:"tr",align:null},"None")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"bufferGrouperMaxLoadFactor")),(0,i.yg)("td",{parentName:"tr",align:null},"Overrides the value of ",(0,i.yg)("inlineCode",{parentName:"td"},"druid.query.groupBy.bufferGrouperMaxLoadFactor")," for this query."),(0,i.yg)("td",{parentName:"tr",align:null},"None")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"forceHashAggregation")),(0,i.yg)("td",{parentName:"tr",align:null},"Overrides the value of ",(0,i.yg)("inlineCode",{parentName:"td"},"druid.query.groupBy.forceHashAggregation")),(0,i.yg)("td",{parentName:"tr",align:null},"None")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"intermediateCombineDegree")),(0,i.yg)("td",{parentName:"tr",align:null},"Overrides the value of ",(0,i.yg)("inlineCode",{parentName:"td"},"druid.query.groupBy.intermediateCombineDegree")),(0,i.yg)("td",{parentName:"tr",align:null},"None")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"numParallelCombineThreads")),(0,i.yg)("td",{parentName:"tr",align:null},"Overrides the value of ",(0,i.yg)("inlineCode",{parentName:"td"},"druid.query.groupBy.numParallelCombineThreads")),(0,i.yg)("td",{parentName:"tr",align:null},"None")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"maxSelectorDictionarySize")),(0,i.yg)("td",{parentName:"tr",align:null},"Overrides the value of ",(0,i.yg)("inlineCode",{parentName:"td"},"druid.query.groupBy.maxMergingDictionarySize")),(0,i.yg)("td",{parentName:"tr",align:null},"None")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"maxMergingDictionarySize")),(0,i.yg)("td",{parentName:"tr",align:null},"Overrides the value of ",(0,i.yg)("inlineCode",{parentName:"td"},"druid.query.groupBy.maxMergingDictionarySize")),(0,i.yg)("td",{parentName:"tr",align:null},"None")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"mergeThreadLocal")),(0,i.yg)("td",{parentName:"tr",align:null},"Whether merge buffers should always be split into thread-local buffers. Setting this to ",(0,i.yg)("inlineCode",{parentName:"td"},"true")," reduces thread contention, but uses memory less efficiently. This tradeoff is beneficial when memory is plentiful."),(0,i.yg)("td",{parentName:"tr",align:null},"false")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"sortByDimsFirst")),(0,i.yg)("td",{parentName:"tr",align:null},"Sort the results first by dimension values and then by timestamp."),(0,i.yg)("td",{parentName:"tr",align:null},"false")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"forceLimitPushDown")),(0,i.yg)("td",{parentName:"tr",align:null},"When all fields in the orderby are part of the grouping key, the Broker will push limit application down to the Historical processes. When the sorting order uses fields that are not in the grouping key, applying this optimization can result in approximate results with unknown accuracy, so this optimization is disabled by default in that case. Enabling this context flag turns on limit push down for limit/orderbys that contain non-grouping key columns."),(0,i.yg)("td",{parentName:"tr",align:null},"false")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"applyLimitPushDownToSegment")),(0,i.yg)("td",{parentName:"tr",align:null},"If Broker pushes limit down to queryable nodes (historicals, peons) then limit results during segment scan. This context value can be used to override ",(0,i.yg)("inlineCode",{parentName:"td"},"druid.query.groupBy.applyLimitPushDownToSegment"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"true")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"groupByEnableMultiValueUnnesting")),(0,i.yg)("td",{parentName:"tr",align:null},"Safety flag to enable/disable the implicit unnesting on multi value column's as part of the grouping key. 'true' indicates multi-value grouping keys are unnested. 'false' returns an error if a multi value column is found as part of the grouping key."),(0,i.yg)("td",{parentName:"tr",align:null},"true")))),(0,i.yg)("h4",{id:"array-based-result-rows"},"Array based result rows"),(0,i.yg)("p",null,"Internally Druid always uses an array based representation of groupBy result rows, but by default this is translated\ninto a map based result format at the Broker. To reduce the overhead of this translation, results may also be returned\nfrom the Broker directly in the array based format if ",(0,i.yg)("inlineCode",{parentName:"p"},"resultAsArray")," is set to ",(0,i.yg)("inlineCode",{parentName:"p"},"true")," on the query context."),(0,i.yg)("p",null,"Each row is positional, and has the following fields, in order:"),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},"Timestamp (optional; only if granularity != ALL)"),(0,i.yg)("li",{parentName:"ul"},"Dimensions (in order)"),(0,i.yg)("li",{parentName:"ul"},"Aggregators (in order)"),(0,i.yg)("li",{parentName:"ul"},"Post-aggregators (optional; in order, if present)")),(0,i.yg)("p",null,"This schema is not available on the response, so it must be computed from the issued query in order to properly read\nthe results."))}m.isMDXComponent=!0}}]); |