| "use strict";(self.webpackChunk=self.webpackChunk||[]).push([[3910],{3905:(e,t,n)=>{n.d(t,{Zo:()=>u,kt:()=>g});var a=n(7294);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function s(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function i(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?s(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):s(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function o(e,t){if(null==e)return{};var n,a,r=function(e,t){if(null==e)return{};var n,a,r={},s=Object.keys(e);for(a=0;a<s.length;a++)n=s[a],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var s=Object.getOwnPropertySymbols(e);for(a=0;a<s.length;a++)n=s[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var l=a.createContext({}),p=function(e){var t=a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},u=function(e){var t=p(e.components);return a.createElement(l.Provider,{value:t},e.children)},d="mdxType",m={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},y=a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,s=e.originalType,l=e.parentName,u=o(e,["components","mdxType","originalType","parentName"]),d=p(n),y=r,g=d["".concat(l,".").concat(y)]||d[y]||m[y]||s;return n?a.createElement(g,i(i({ref:t},u),{},{components:n})):a.createElement(g,i({ref:t},u))}));function g(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var s=n.length,i=new Array(s);i[0]=y;var o={};for(var l in t)hasOwnProperty.call(t,l)&&(o[l]=t[l]);o.originalType=e,o[d]="string"==typeof e?e:r,i[1]=o;for(var p=2;p<s;p++)i[p]=n[p];return a.createElement.apply(null,i)}return a.createElement.apply(null,n)}y.displayName="MDXCreateElement"},9077:(e,t,n)=>{n.r(t),n.d(t,{assets:()=>u,contentTitle:()=>l,default:()=>g,frontMatter:()=>o,metadata:()=>p,toc:()=>d});var a=n(7462),r=n(3366),s=(n(7294),n(3905)),i=["components"],o={id:"api",title:"SQL-based ingestion and multi-stage query task API",sidebar_label:"API"},l=void 0,p={unversionedId:"multi-stage-query/api",id:"multi-stage-query/api",title:"SQL-based ingestion and multi-stage query task API",description:"\x3c!--",source:"@site/docs/latest/multi-stage-query/api.md",sourceDirName:"multi-stage-query",slug:"/multi-stage-query/api",permalink:"/docs/latest/multi-stage-query/api",draft:!1,tags:[],version:"current",frontMatter:{id:"api",title:"SQL-based ingestion and multi-stage query task API",sidebar_label:"API"},sidebar:"docs",previous:{title:"Key concepts",permalink:"/docs/latest/multi-stage-query/concepts"},next:{title:"Security",permalink:"/docs/latest/multi-stage-query/security"}},u={},d=[{value:"Submit a query",id:"submit-a-query",level:2},{value:"Request",id:"request",level:4},{value:"Response",id:"response",level:4},{value:"Get the status for a query task",id:"get-the-status-for-a-query-task",level:2},{value:"Request",id:"request-1",level:4},{value:"Response",id:"response-1",level:4},{value:"Get the report for a query task",id:"get-the-report-for-a-query-task",level:2},{value:"Request",id:"request-2",level:4},{value:"Response",id:"response-2",level:4},{value:"Cancel a query task",id:"cancel-a-query-task",level:2},{value:"Request",id:"request-3",level:4},{value:"Response",id:"response-3",level:4}],m={toc:d},y="wrapper";function g(e){var t=e.components,n=(0,r.Z)(e,i);return(0,s.kt)(y,(0,a.Z)({},m,n,{components:t,mdxType:"MDXLayout"}),(0,s.kt)("blockquote",null,(0,s.kt)("p",{parentName:"blockquote"},"This page describes SQL-based batch ingestion using the ",(0,s.kt)("a",{parentName:"p",href:"/docs/latest/multi-stage-query/"},(0,s.kt)("inlineCode",{parentName:"a"},"druid-multi-stage-query")),"\nextension, new in Druid 24.0. Refer to the ",(0,s.kt)("a",{parentName:"p",href:"/docs/latest/ingestion/#batch"},"ingestion methods")," table to determine which\ningestion method is right for you.")),(0,s.kt)("p",null,"The ",(0,s.kt)("strong",{parentName:"p"},"Query")," view in the web console provides a friendly experience for the multi-stage query task engine (MSQ task\nengine) and multi-stage query architecture. We recommend using the web console if you do not need a programmatic\ninterface."),(0,s.kt)("p",null,"When using the API for the MSQ task engine, the action you want to take determines the endpoint you use:"),(0,s.kt)("ul",null,(0,s.kt)("li",{parentName:"ul"},(0,s.kt)("inlineCode",{parentName:"li"},"/druid/v2/sql/task")," endpoint: Submit a query for ingestion."),(0,s.kt)("li",{parentName:"ul"},(0,s.kt)("inlineCode",{parentName:"li"},"/druid/indexer/v1/task")," endpoint: Interact with a query, including getting its status, getting its details, or canceling it. This page describes a few of the Overlord Task APIs that you can use with the MSQ task engine. For information about Druid APIs, see the ",(0,s.kt)("a",{parentName:"li",href:"/docs/latest/operations/api-reference#tasks"},"API reference for Druid"),".")),(0,s.kt)("h2",{id:"submit-a-query"},"Submit a query"),(0,s.kt)("p",null,"You submit queries to the MSQ task engine using the ",(0,s.kt)("inlineCode",{parentName:"p"},"POST /druid/v2/sql/task/")," endpoint."),(0,s.kt)("h4",{id:"request"},"Request"),(0,s.kt)("p",null,"The SQL task endpoint accepts ",(0,s.kt)("a",{parentName:"p",href:"/docs/latest/querying/sql-api#request-body"},"SQL requests in the JSON-over-HTTP form")," using the\n",(0,s.kt)("inlineCode",{parentName:"p"},"query"),", ",(0,s.kt)("inlineCode",{parentName:"p"},"context"),", and ",(0,s.kt)("inlineCode",{parentName:"p"},"parameters")," fields, but ignoring the ",(0,s.kt)("inlineCode",{parentName:"p"},"resultFormat"),", ",(0,s.kt)("inlineCode",{parentName:"p"},"header"),", ",(0,s.kt)("inlineCode",{parentName:"p"},"typesHeader"),", and\n",(0,s.kt)("inlineCode",{parentName:"p"},"sqlTypesHeader")," fields."),(0,s.kt)("p",null,"This endpoint accepts ",(0,s.kt)("a",{parentName:"p",href:"/docs/latest/multi-stage-query/reference#insert"},"INSERT")," and ",(0,s.kt)("a",{parentName:"p",href:"/docs/latest/multi-stage-query/reference#replace"},"REPLACE")," statements."),(0,s.kt)("p",null,"As an experimental feature, this endpoint also accepts SELECT queries. SELECT query results are collected from workers\nby the controller, and written into the ",(0,s.kt)("a",{parentName:"p",href:"#get-the-report-for-a-query-task"},"task report")," as an array of arrays. The\nbehavior and result format of plain SELECT queries (without INSERT or REPLACE) is subject to change."),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre"},"POST /druid/v2/sql/task\n")),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-json"},'{\n "query": "INSERT INTO wikipedia\\nSELECT\\n TIME_PARSE(\\"timestamp\\") AS __time,\\n *\\nFROM TABLE(\\n EXTERN(\\n \'{\\"type\\": \\"http\\", \\"uris\\": [\\"https://druid.apache.org/data/wikipedia.json.gz\\"]}\',\\n \'{\\"type\\": \\"json\\"}\',\\n \'[{\\"name\\": \\"added\\", \\"type\\": \\"long\\"}, {\\"name\\": \\"channel\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"cityName\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"comment\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"commentLength\\", \\"type\\": \\"long\\"}, {\\"name\\": \\"countryIsoCode\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"countryName\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"deleted\\", \\"type\\": \\"long\\"}, {\\"name\\": \\"delta\\", \\"type\\": \\"long\\"}, {\\"name\\": \\"deltaBucket\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"diffUrl\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"flags\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"isAnonymous\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"isMinor\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"isNew\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"isRobot\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"isUnpatrolled\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"metroCode\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"namespace\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"page\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"regionIsoCode\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"regionName\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"timestamp\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"user\\", \\"type\\": \\"string\\"}]\'\\n )\\n)\\nPARTITIONED BY DAY",\n "context": {\n "maxNumTasks": 3\n }\n}\n')),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-bash"},'# Make sure you replace `username`, `password`, `your-instance`, and `port` with the values for your deployment.\ncurl --location --request POST \'https://<username>:<password>@<your-instance>:<port>/druid/v2/sql/task/\' \\\n --header \'Content-Type: application/json\' \\\n --data-raw \'{\n "query": "INSERT INTO wikipedia\\nSELECT\\n TIME_PARSE(\\"timestamp\\") AS __time,\\n *\\nFROM TABLE(\\n EXTERN(\\n \'\\\'\'{\\"type\\": \\"http\\", \\"uris\\": [\\"https://druid.apache.org/data/wikipedia.json.gz\\"]}\'\\\'\',\\n \'\\\'\'{\\"type\\": \\"json\\"}\'\\\'\',\\n \'\\\'\'[{\\"name\\": \\"added\\", \\"type\\": \\"long\\"}, {\\"name\\": \\"channel\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"cityName\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"comment\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"commentLength\\", \\"type\\": \\"long\\"}, {\\"name\\": \\"countryIsoCode\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"countryName\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"deleted\\", \\"type\\": \\"long\\"}, {\\"name\\": \\"delta\\", \\"type\\": \\"long\\"}, {\\"name\\": \\"deltaBucket\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"diffUrl\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"flags\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"isAnonymous\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"isMinor\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"isNew\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"isRobot\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"isUnpatrolled\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"metroCode\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"namespace\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"page\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"regionIsoCode\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"regionName\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"timestamp\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"user\\", \\"type\\": \\"string\\"}]\'\\\'\'\\n )\\n)\\nPARTITIONED BY DAY",\n "context": {\n "maxNumTasks": 3\n }\n }\'\n')),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-python"},'import json\nimport requests\n\n# Make sure you replace `your-instance`, and `port` with the values for your deployment.\nurl = "https://<your-instance>:<port>/druid/v2/sql/task/"\n\npayload = json.dumps({\n "query": "INSERT INTO wikipedia\\nSELECT\\n TIME_PARSE(\\"timestamp\\") AS __time,\\n *\\nFROM TABLE(\\n EXTERN(\\n \'{\\"type\\": \\"http\\", \\"uris\\": [\\"https://druid.apache.org/data/wikipedia.json.gz\\"]}\',\\n \'{\\"type\\": \\"json\\"}\',\\n \'[{\\"name\\": \\"added\\", \\"type\\": \\"long\\"}, {\\"name\\": \\"channel\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"cityName\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"comment\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"commentLength\\", \\"type\\": \\"long\\"}, {\\"name\\": \\"countryIsoCode\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"countryName\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"deleted\\", \\"type\\": \\"long\\"}, {\\"name\\": \\"delta\\", \\"type\\": \\"long\\"}, {\\"name\\": \\"deltaBucket\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"diffUrl\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"flags\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"isAnonymous\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"isMinor\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"isNew\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"isRobot\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"isUnpatrolled\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"metroCode\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"namespace\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"page\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"regionIsoCode\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"regionName\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"timestamp\\", \\"type\\": \\"string\\"}, {\\"name\\": \\"user\\", \\"type\\": \\"string\\"}]\'\\n )\\n)\\nPARTITIONED BY DAY",\n "context": {\n "maxNumTasks": 3\n }\n})\nheaders = {\n \'Content-Type\': \'application/json\'\n}\n\nresponse = requests.post(url, headers=headers, data=payload, auth=(\'USER\', \'PASSWORD\'))\n\nprint(response.text)\n\n')),(0,s.kt)("h4",{id:"response"},"Response"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-json"},'{\n "taskId": "query-f795a235-4dc7-4fef-abac-3ae3f9686b79",\n "state": "RUNNING",\n}\n')),(0,s.kt)("p",null,(0,s.kt)("strong",{parentName:"p"},"Response fields")),(0,s.kt)("table",null,(0,s.kt)("thead",{parentName:"table"},(0,s.kt)("tr",{parentName:"thead"},(0,s.kt)("th",{parentName:"tr",align:null},"Field"),(0,s.kt)("th",{parentName:"tr",align:null},"Description"))),(0,s.kt)("tbody",{parentName:"table"},(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"taskId")),(0,s.kt)("td",{parentName:"tr",align:null},"Controller task ID. You can use Druid's standard ",(0,s.kt)("a",{parentName:"td",href:"/docs/latest/operations/api-reference#overlord"},"task APIs")," to interact with this controller task.")),(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"state")),(0,s.kt)("td",{parentName:"tr",align:null},'Initial state for the query, which is "RUNNING".')))),(0,s.kt)("h2",{id:"get-the-status-for-a-query-task"},"Get the status for a query task"),(0,s.kt)("p",null,"You can retrieve status of a query to see if it is still running, completed successfully, failed, or got canceled."),(0,s.kt)("h4",{id:"request-1"},"Request"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre"},"GET /druid/indexer/v1/task/<taskId>/status\n")),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-bash"},"# Make sure you replace `username`, `password`, `your-instance`, `port`, and `taskId` with the values for your deployment.\ncurl --location --request GET 'https://<username>:<password>@<your-instance>:<port>/druid/indexer/v1/task/<taskId>/status'\n")),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-python"},"import requests\n\n# Make sure you replace `your-instance`, `port`, and `taskId` with the values for your deployment.\nurl = \"https://<your-instance>:<port>/druid/indexer/v1/task/<taskId>/status\"\n\npayload={}\nheaders = {}\n\nresponse = requests.get(url, headers=headers, data=payload, auth=('USER', 'PASSWORD'))\n\nprint(response.text)\n")),(0,s.kt)("h4",{id:"response-1"},"Response"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre"},'{\n "task": "query-3dc0c45d-34d7-4b15-86c9-cdb2d3ebfc4e",\n "status": {\n "id": "query-3dc0c45d-34d7-4b15-86c9-cdb2d3ebfc4e",\n "groupId": "query-3dc0c45d-34d7-4b15-86c9-cdb2d3ebfc4e",\n "type": "query_controller",\n "createdTime": "2022-09-14T22:12:00.183Z",\n "queueInsertionTime": "1970-01-01T00:00:00.000Z",\n "statusCode": "RUNNING",\n "status": "RUNNING",\n "runnerStatusCode": "RUNNING",\n "duration": -1,\n "location": {\n "host": "localhost",\n "port": 8100,\n "tlsPort": -1\n },\n "dataSource": "kttm_simple",\n "errorMsg": null\n }\n}\n')),(0,s.kt)("h2",{id:"get-the-report-for-a-query-task"},"Get the report for a query task"),(0,s.kt)("p",null,"A report provides detailed information about a query task, including things like the stages, warnings, and errors."),(0,s.kt)("p",null,"Keep the following in mind when using the task API to view reports:"),(0,s.kt)("ul",null,(0,s.kt)("li",{parentName:"ul"},"The task report for an entire job is associated with the ",(0,s.kt)("inlineCode",{parentName:"li"},"query_controller")," task. The ",(0,s.kt)("inlineCode",{parentName:"li"},"query_worker")," tasks do not have\ntheir own reports; their information is incorporated into the controller report."),(0,s.kt)("li",{parentName:"ul"},"The task report API may report ",(0,s.kt)("inlineCode",{parentName:"li"},"404 Not Found")," temporarily while the task is in the process of starting up."),(0,s.kt)("li",{parentName:"ul"},"As an experimental feature, the MSQ task engine supports running SELECT queries. SELECT query results are written into\nthe ",(0,s.kt)("inlineCode",{parentName:"li"},"multiStageQuery.payload.results.results")," task report key as an array of arrays. The behavior and result format of plain\nSELECT queries (without INSERT or REPLACE) is subject to change.")),(0,s.kt)("p",null,"For an explanation of the fields in a report, see ",(0,s.kt)("a",{parentName:"p",href:"#report-response-fields"},"Report response fields"),"."),(0,s.kt)("h4",{id:"request-2"},"Request"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre"},"GET /druid/indexer/v1/task/<taskId>/reports\n")),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-bash"},"# Make sure you replace `username`, `password`, `your-instance`, `port`, and `taskId` with the values for your deployment.\ncurl --location --request GET 'https://<username>:<password>@<your-instance>:<port>/druid/indexer/v1/task/<taskId>/reports'\n")),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-python"},"import requests\n\n# Make sure you replace `your-instance`, `port`, and `taskId` with the values for your deployment.\nurl = \"https://<your-instance>:<port>/druid/indexer/v1/task/<taskId>/reports\"\n\nheaders = {}\nresponse = requests.get(url, headers=headers, auth=('USER', 'PASSWORD'))\nprint(response.text)\n")),(0,s.kt)("h4",{id:"response-2"},"Response"),(0,s.kt)("p",null,"The response shows an example report for a query."),(0,s.kt)("details",null,(0,s.kt)("summary",null,"Show the response"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-json"},'{\n "multiStageQuery": {\n "type": "multiStageQuery",\n "taskId": "query-3dc0c45d-34d7-4b15-86c9-cdb2d3ebfc4e",\n "payload": {\n "status": {\n "status": "SUCCESS",\n "startTime": "2022-09-14T22:12:09.266Z",\n "durationMs": 28227,\n "pendingTasks": 0,\n "runningTasks": 2\n },\n "stages": [\n {\n "stageNumber": 0,\n "definition": {\n "id": "71ecb11e-09d7-42f8-9225-1662c8e7e121_0",\n "input": [\n {\n "type": "external",\n "inputSource": {\n "type": "http",\n "uris": [\n "https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"\n ],\n "httpAuthenticationUsername": null,\n "httpAuthenticationPassword": null\n },\n "inputFormat": {\n "type": "json",\n "flattenSpec": null,\n "featureSpec": {},\n "keepNullColumns": false\n },\n "signature": [\n {\n "name": "timestamp",\n "type": "STRING"\n },\n {\n "name": "agent_category",\n "type": "STRING"\n },\n {\n "name": "agent_type",\n "type": "STRING"\n }\n ]\n }\n ],\n "processor": {\n "type": "scan",\n "query": {\n "queryType": "scan",\n "dataSource": {\n "type": "inputNumber",\n "inputNumber": 0\n },\n "intervals": {\n "type": "intervals",\n "intervals": [\n "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z"\n ]\n },\n "resultFormat": "compactedList",\n "columns": [\n "agent_category",\n "agent_type",\n "timestamp"\n ],\n "legacy": false,\n "context": {\n "finalize": false,\n "finalizeAggregations": false,\n "groupByEnableMultiValueUnnesting": false,\n "scanSignature": "[{\\"name\\":\\"agent_category\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"agent_type\\",\\"type\\":\\"STRING\\"},{\\"name\\":\\"timestamp\\",\\"type\\":\\"STRING\\"}]",\n "sqlInsertSegmentGranularity": "{\\"type\\":\\"all\\"}",\n "sqlQueryId": "3dc0c45d-34d7-4b15-86c9-cdb2d3ebfc4e",\n "sqlReplaceTimeChunks": "all"\n },\n "granularity": {\n "type": "all"\n }\n }\n },\n "signature": [\n {\n "name": "__boost",\n "type": "LONG"\n },\n {\n "name": "agent_category",\n "type": "STRING"\n },\n {\n "name": "agent_type",\n "type": "STRING"\n },\n {\n "name": "timestamp",\n "type": "STRING"\n }\n ],\n "shuffleSpec": {\n "type": "targetSize",\n "clusterBy": {\n "columns": [\n {\n "columnName": "__boost"\n }\n ]\n },\n "targetSize": 3000000\n },\n "maxWorkerCount": 1,\n "shuffleCheckHasMultipleValues": true\n },\n "phase": "FINISHED",\n "workerCount": 1,\n "partitionCount": 1,\n "startTime": "2022-09-14T22:12:11.663Z",\n "duration": 19965,\n "sort": true\n },\n {\n "stageNumber": 1,\n "definition": {\n "id": "71ecb11e-09d7-42f8-9225-1662c8e7e121_1",\n "input": [\n {\n "type": "stage",\n "stage": 0\n }\n ],\n "processor": {\n "type": "segmentGenerator",\n "dataSchema": {\n "dataSource": "kttm_simple",\n "timestampSpec": {\n "column": "__time",\n "format": "millis",\n "missingValue": null\n },\n "dimensionsSpec": {\n "dimensions": [\n {\n "type": "string",\n "name": "timestamp",\n "multiValueHandling": "SORTED_ARRAY",\n "createBitmapIndex": true\n },\n {\n "type": "string",\n "name": "agent_category",\n "multiValueHandling": "SORTED_ARRAY",\n "createBitmapIndex": true\n },\n {\n "type": "string",\n "name": "agent_type",\n "multiValueHandling": "SORTED_ARRAY",\n "createBitmapIndex": true\n }\n ],\n "dimensionExclusions": [\n "__time"\n ],\n "includeAllDimensions": false\n },\n "metricsSpec": [],\n "granularitySpec": {\n "type": "arbitrary",\n "queryGranularity": {\n "type": "none"\n },\n "rollup": false,\n "intervals": [\n "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z"\n ]\n },\n "transformSpec": {\n "filter": null,\n "transforms": []\n }\n },\n "columnMappings": [\n {\n "queryColumn": "timestamp",\n "outputColumn": "timestamp"\n },\n {\n "queryColumn": "agent_category",\n "outputColumn": "agent_category"\n },\n {\n "queryColumn": "agent_type",\n "outputColumn": "agent_type"\n }\n ],\n "tuningConfig": {\n "maxNumWorkers": 1,\n "maxRowsInMemory": 100000,\n "rowsPerSegment": 3000000\n }\n },\n "signature": [],\n "maxWorkerCount": 1\n },\n "phase": "FINISHED",\n "workerCount": 1,\n "partitionCount": 1,\n "startTime": "2022-09-14T22:12:31.602Z",\n "duration": 5891\n }\n ],\n "counters": {\n "0": {\n "0": {\n "input0": {\n "type": "channel",\n "rows": [\n 465346\n ],\n "files": [\n 1\n ],\n "totalFiles": [\n 1\n ]\n },\n "output": {\n "type": "channel",\n "rows": [\n 465346\n ],\n "bytes": [\n 43694447\n ],\n "frames": [\n 7\n ]\n },\n "shuffle": {\n "type": "channel",\n "rows": [\n 465346\n ],\n "bytes": [\n 41835307\n ],\n "frames": [\n 73\n ]\n },\n "sortProgress": {\n "type": "sortProgress",\n "totalMergingLevels": 3,\n "levelToTotalBatches": {\n "0": 1,\n "1": 1,\n "2": 1\n },\n "levelToMergedBatches": {\n "0": 1,\n "1": 1,\n "2": 1\n }, \n "totalMergersForUltimateLevel": 1,\n "progressDigest": 1\n }\n }\n },\n "1": {\n "0": {\n "input0": {\n "type": "channel",\n "rows": [\n 465346\n ],\n "bytes": [\n 41835307\n ],\n "frames": [\n 73\n ]\n },\n "segmentGenerationProgress": {\n "type": "segmentGenerationProgress",\n "rowsProcessed": 465346,\n "rowsPersisted": 465346,\n "rowsMerged": 465346\n }\n }\n }\n }\n }\n }\n}\n'))),(0,s.kt)("a",{name:"report-response-fields"}),(0,s.kt)("p",null,"The following table describes the response fields when you retrieve a report for a MSQ task engine using the ",(0,s.kt)("inlineCode",{parentName:"p"},"/druid/indexer/v1/task/<taskId>/reports")," endpoint:"),(0,s.kt)("table",null,(0,s.kt)("thead",{parentName:"table"},(0,s.kt)("tr",{parentName:"thead"},(0,s.kt)("th",{parentName:"tr",align:null},"Field"),(0,s.kt)("th",{parentName:"tr",align:null},"Description"))),(0,s.kt)("tbody",{parentName:"table"},(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"multiStageQuery.taskId")),(0,s.kt)("td",{parentName:"tr",align:null},"Controller task ID.")),(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"multiStageQuery.payload.status")),(0,s.kt)("td",{parentName:"tr",align:null},"Query status container.")),(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"multiStageQuery.payload.status.status")),(0,s.kt)("td",{parentName:"tr",align:null},"RUNNING, SUCCESS, or FAILED.")),(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"multiStageQuery.payload.status.startTime")),(0,s.kt)("td",{parentName:"tr",align:null},"Start time of the query in ISO format. Only present if the query has started running.")),(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"multiStageQuery.payload.status.durationMs")),(0,s.kt)("td",{parentName:"tr",align:null},"Milliseconds elapsed after the query has started running. -1 denotes that the query hasn't started running yet.")),(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"multiStageQuery.payload.status.pendingTasks")),(0,s.kt)("td",{parentName:"tr",align:null},"Number of tasks that are not fully started. -1 denotes that the number is currently unknown.")),(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"multiStageQuery.payload.status.runningTasks")),(0,s.kt)("td",{parentName:"tr",align:null},"Number of currently running tasks. Should be at least 1 since the controller is included.")),(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"multiStageQuery.payload.status.errorReport")),(0,s.kt)("td",{parentName:"tr",align:null},"Error object. Only present if there was an error.")),(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"multiStageQuery.payload.status.errorReport.taskId")),(0,s.kt)("td",{parentName:"tr",align:null},"The task that reported the error, if known. May be a controller task or a worker task.")),(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"multiStageQuery.payload.status.errorReport.host")),(0,s.kt)("td",{parentName:"tr",align:null},"The hostname and port of the task that reported the error, if known.")),(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"multiStageQuery.payload.status.errorReport.stageNumber")),(0,s.kt)("td",{parentName:"tr",align:null},"The stage number that reported the error, if it happened during execution of a specific stage.")),(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"multiStageQuery.payload.status.errorReport.error")),(0,s.kt)("td",{parentName:"tr",align:null},"Error object. Contains ",(0,s.kt)("inlineCode",{parentName:"td"},"errorCode")," at a minimum, and may contain other fields as described in the ",(0,s.kt)("a",{parentName:"td",href:"/docs/latest/multi-stage-query/reference#error-codes"},"error code table"),". Always present if there is an error.")),(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"multiStageQuery.payload.status.errorReport.error.errorCode")),(0,s.kt)("td",{parentName:"tr",align:null},"One of the error codes from the ",(0,s.kt)("a",{parentName:"td",href:"/docs/latest/multi-stage-query/reference#error-codes"},"error code table"),". Always present if there is an error.")),(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"multiStageQuery.payload.status.errorReport.error.errorMessage")),(0,s.kt)("td",{parentName:"tr",align:null},"User-friendly error message. Not always present, even if there is an error.")),(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"multiStageQuery.payload.status.errorReport.exceptionStackTrace")),(0,s.kt)("td",{parentName:"tr",align:null},"Java stack trace in string form, if the error was due to a server-side exception.")),(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"multiStageQuery.payload.stages")),(0,s.kt)("td",{parentName:"tr",align:null},"Array of query stages.")),(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"multiStageQuery.payload.stages[].stageNumber")),(0,s.kt)("td",{parentName:"tr",align:null},"Each stage has a number that differentiates it from other stages.")),(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"multiStageQuery.payload.stages[].phase")),(0,s.kt)("td",{parentName:"tr",align:null},"Either NEW, READING_INPUT, POST_READING, RESULTS_COMPLETE, or FAILED. Only present if the stage has started.")),(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"multiStageQuery.payload.stages[].workerCount")),(0,s.kt)("td",{parentName:"tr",align:null},"Number of parallel tasks that this stage is running on. Only present if the stage has started.")),(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"multiStageQuery.payload.stages[].partitionCount")),(0,s.kt)("td",{parentName:"tr",align:null},"Number of output partitions generated by this stage. Only present if the stage has started and has computed its number of output partitions.")),(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"multiStageQuery.payload.stages[].startTime")),(0,s.kt)("td",{parentName:"tr",align:null},"Start time of this stage. Only present if the stage has started.")),(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"multiStageQuery.payload.stages[].duration")),(0,s.kt)("td",{parentName:"tr",align:null},"The number of milliseconds that the stage has been running. Only present if the stage has started.")),(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"multiStageQuery.payload.stages[].sort")),(0,s.kt)("td",{parentName:"tr",align:null},"A boolean that is set to ",(0,s.kt)("inlineCode",{parentName:"td"},"true")," if the stage does a sort as part of its execution.")),(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"multiStageQuery.payload.stages[].definition")),(0,s.kt)("td",{parentName:"tr",align:null},"The object defining what the stage does.")),(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"multiStageQuery.payload.stages[].definition.id")),(0,s.kt)("td",{parentName:"tr",align:null},"The unique identifier of the stage.")),(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"multiStageQuery.payload.stages[].definition.input")),(0,s.kt)("td",{parentName:"tr",align:null},"Array of inputs that the stage has.")),(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"multiStageQuery.payload.stages[].definition.broadcast")),(0,s.kt)("td",{parentName:"tr",align:null},"Array of input indexes that get broadcasted. Only present if there are inputs that get broadcasted.")),(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"multiStageQuery.payload.stages[].definition.processor")),(0,s.kt)("td",{parentName:"tr",align:null},"An object defining the processor logic.")),(0,s.kt)("tr",{parentName:"tbody"},(0,s.kt)("td",{parentName:"tr",align:null},(0,s.kt)("inlineCode",{parentName:"td"},"multiStageQuery.payload.stages[].definition.signature")),(0,s.kt)("td",{parentName:"tr",align:null},"The output signature of the stage.")))),(0,s.kt)("h2",{id:"cancel-a-query-task"},"Cancel a query task"),(0,s.kt)("h4",{id:"request-3"},"Request"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre"},"POST /druid/indexer/v1/task/<taskId>/shutdown\n")),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-bash"},"# Make sure you replace `username`, `password`, `your-instance`, `port`, and `taskId` with the values for your deployment.\ncurl --location --request POST 'https://<username>:<password>@<your-instance>:<port>/druid/indexer/v1/task/<taskId>/shutdown'\n")),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-python"},"import requests\n\n# Make sure you replace `your-instance`, `port`, and `taskId` with the values for your deployment.\nurl = \"https://<your-instance>:<port>/druid/indexer/v1/task/<taskId>/shutdown\"\n\npayload={}\nheaders = {}\n\nresponse = requests.post(url, headers=headers, data=payload, auth=('USER', 'PASSWORD'))\n\nprint(response.text)\n")),(0,s.kt)("h4",{id:"response-3"},"Response"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre"},'{\n "task": "query-655efe33-781a-4c50-ae84-c2911b42d63c"\n}\n')))}g.isMDXComponent=!0}}]); |