"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[6346],{3905:(e,t,n)=>{n.d(t,{Zo:()=>m,kt:()=>g});var a=n(67294);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function i(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?o(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):o(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,a,r=function(e,t){if(null==e)return{};var n,a,r={},o=Object.keys(e);for(a=0;a<o.length;a++)n=o[a],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a<o.length;a++)n=o[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var l=a.createContext({}),p=function(e){var t=a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},m=function(e){var t=p(e.components);return a.createElement(l.Provider,{value:t},e.children)},u="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},c=a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,o=e.originalType,l=e.parentName,m=s(e,["components","mdxType","originalType","parentName"]),u=p(n),c=r,g=u["".concat(l,".").concat(c)]||u[c]||d[c]||o;return n?a.createElement(g,i(i({ref:t},m),{},{components:n})):a.createElement(g,i({ref:t},m))}));function g(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=n.length,i=new Array(o);i[0]=c;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s[u]="string"==typeof e?e:r,i[1]=s;for(var p=2;p<o;p++)i[p]=n[p];return a.createElement.apply(null,i)}return a.createElement.apply(null,n)}c.displayName="MDXCreateElement"},19096:(e,t,n)=>{n.r(t),n.d(t,{assets:()=>m,contentTitle:()=>l,default:()=>g,frontMatter:()=>s,metadata:()=>p,toc:()=>u});var a=n(87462),r=n(63366),o=(n(67294),n(3905)),i=["components"],s={id:"tutorial-msq-extern",title:"Load files with SQL-based ingestion",sidebar_label:"Load files using SQL",description:"How to generate a query that references externally hosted data"},l=void 0,p={unversionedId:"tutorials/tutorial-msq-extern",id:"tutorials/tutorial-msq-extern",title:"Load files with SQL-based ingestion",description:"How to generate a query that references externally hosted data",source:"@site/docs/28.0.0/tutorials/tutorial-msq-extern.md",sourceDirName:"tutorials",slug:"/tutorials/tutorial-msq-extern",permalink:"/docs/28.0.0/tutorials/tutorial-msq-extern",draft:!1,tags:[],version:"current",frontMatter:{id:"tutorial-msq-extern",title:"Load files with SQL-based ingestion",sidebar_label:"Load files using SQL",description:"How to generate a query that references externally hosted data"},sidebar:"docs",previous:{title:"Clustered deployment",permalink:"/docs/28.0.0/tutorials/cluster"},next:{title:"Load from Apache Kafka",permalink:"/docs/28.0.0/tutorials/tutorial-kafka"}},m={},u=[{value:"Query the data",id:"query-the-data",level:2},{value:"Further reading",id:"further-reading",level:2}],d={toc:u},c="wrapper";function g(e){var t=e.components,n=(0,r.Z)(e,i);return(0,o.kt)(c,(0,a.Z)({},d,n,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("admonition",{type:"info"},(0,o.kt)("p",{parentName:"admonition"}," This page describes SQL-based batch ingestion using the ",(0,o.kt)("a",{parentName:"p",href:"/docs/28.0.0/multi-stage-query/"},(0,o.kt)("inlineCode",{parentName:"a"},"druid-multi-stage-query")),"\nextension, new in Druid 24.0. Refer to the ",(0,o.kt)("a",{parentName:"p",href:"/docs/28.0.0/ingestion/#batch"},"ingestion methods")," table to determine which\ningestion method is right for you.")),(0,o.kt)("p",null,"This tutorial demonstrates how to generate a query that references externally hosted data using the ",(0,o.kt)("strong",{parentName:"p"},"Connect external data")," wizard."),(0,o.kt)("p",null,"The following example uses EXTERN to query a JSON file located at ",(0,o.kt)("a",{parentName:"p",href:"https://druid.apache.org/data/wikipedia.json.gz"},"https://druid.apache.org/data/wikipedia.json.gz"),"."),(0,o.kt)("p",null,"Although you can manually create a query in the UI, you can use Druid to generate a base query for you that you can modify to meet your requirements."),(0,o.kt)("p",null,"To generate a query from external data, do the following:"),(0,o.kt)("ol",null,(0,o.kt)("li",{parentName:"ol"},(0,o.kt)("p",{parentName:"li"},"In the ",(0,o.kt)("strong",{parentName:"p"},"Query")," view of the web console, click ",(0,o.kt)("strong",{parentName:"p"},"Connect external data"),".")),(0,o.kt)("li",{parentName:"ol"},(0,o.kt)("p",{parentName:"li"},"On the ",(0,o.kt)("strong",{parentName:"p"},"Select input type")," screen, choose ",(0,o.kt)("strong",{parentName:"p"},"HTTP(s)")," and enter the following value in the ",(0,o.kt)("strong",{parentName:"p"},"URIs")," field: ",(0,o.kt)("inlineCode",{parentName:"p"},"https://druid.apache.org/data/wikipedia.json.gz"),". Leave the HTTP auth username and password blank.")),(0,o.kt)("li",{parentName:"ol"},(0,o.kt)("p",{parentName:"li"},"Click ",(0,o.kt)("strong",{parentName:"p"},"Connect data"),".")),(0,o.kt)("li",{parentName:"ol"},(0,o.kt)("p",{parentName:"li"},"On the ",(0,o.kt)("strong",{parentName:"p"},"Parse")," screen, you can perform additional actions before you load the data into Druid:"),(0,o.kt)("ul",{parentName:"li"},(0,o.kt)("li",{parentName:"ul"},"Expand a row to see what data it corresponds to from the source."),(0,o.kt)("li",{parentName:"ul"},"Customize how Druid handles the data by selecting the ",(0,o.kt)("strong",{parentName:"li"},"Input format")," and its related options, such as adding ",(0,o.kt)("strong",{parentName:"li"},"JSON parser features")," for JSON files."))),(0,o.kt)("li",{parentName:"ol"},(0,o.kt)("p",{parentName:"li"},"When you're ready, click ",(0,o.kt)("strong",{parentName:"p"},"Done"),". You're returned to the ",(0,o.kt)("strong",{parentName:"p"},"Query")," view where you can see the starter query that will insert the data from the external source into a table named ",(0,o.kt)("inlineCode",{parentName:"p"},"wikipedia"),"."),(0,o.kt)("details",null,(0,o.kt)("summary",null,"Show the query"),(0,o.kt)("pre",{parentName:"li"},(0,o.kt)("code",{parentName:"pre",className:"language-sql"},'REPLACE INTO "wikipedia" OVERWRITE ALL\nWITH ext AS (SELECT *\nFROM TABLE(\n  EXTERN(\n    \'{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}\',\n    \'{"type":"json"}\',\n    \'[{"name":"isRobot","type":"string"},{"name":"channel","type":"string"},{"name":"timestamp","type":"string"},{"name":"flags","type":"string"},{"name":"isUnpatrolled","type":"string"},{"name":"page","type":"string"},{"name":"diffUrl","type":"string"},{"name":"added","type":"long"},{"name":"comment","type":"string"},{"name":"commentLength","type":"long"},{"name":"isNew","type":"string"},{"name":"isMinor","type":"string"},{"name":"delta","type":"long"},{"name":"isAnonymous","type":"string"},{"name":"user","type":"string"},{"name":"deltaBucket","type":"long"},{"name":"deleted","type":"long"},{"name":"namespace","type":"string"},{"name":"cityName","type":"string"},{"name":"countryName","type":"string"},{"name":"regionIsoCode","type":"string"},{"name":"metroCode","type":"long"},{"name":"countryIsoCode","type":"string"},{"name":"regionName","type":"string"}]\'\n  )\n))\nSELECT\n  TIME_PARSE("timestamp") AS __time,\n  isRobot,\n  channel,\n  flags,\n  isUnpatrolled,\n  page,\n  diffUrl,\n  added,\n  comment,\n  commentLength,\n  isNew,\n  isMinor,\n  delta,\n  isAnonymous,\n  user,\n  deltaBucket,\n  deleted,\n  namespace,\n  cityName,\n  countryName,\n  regionIsoCode,\n  metroCode,\n  countryIsoCode,\n  regionName\nFROM ext\nPARTITIONED BY DAY\n')))),(0,o.kt)("li",{parentName:"ol"},(0,o.kt)("p",{parentName:"li"},"Review and modify the query to meet your needs. For example, you can rename the table or change segment granularity. To partition by something other than ALL, include ",(0,o.kt)("inlineCode",{parentName:"p"},'TIME_PARSE("timestamp") AS __time')," in your SELECT statement."),(0,o.kt)("p",{parentName:"li"},"For example, to specify day-based segment granularity, change the partitioning to ",(0,o.kt)("inlineCode",{parentName:"p"},"PARTITIONED BY DAY"),":"),(0,o.kt)("pre",{parentName:"li"},(0,o.kt)("code",{parentName:"pre",className:"language-sql"},' INSERT INTO ...\n SELECT\n   TIME_PARSE("timestamp") AS __time,\n ...\n ...\n PARTITIONED BY DAY\n'))),(0,o.kt)("li",{parentName:"ol"},(0,o.kt)("p",{parentName:"li"},"Optionally, select ",(0,o.kt)("strong",{parentName:"p"},"Preview")," to review the data before you ingest it. A preview runs the query without the REPLACE INTO clause and with an added LIMIT.\nYou can see the general shape of the data before you commit to inserting it.\nThe LIMITs make the query run faster but can cause incomplete results.")),(0,o.kt)("li",{parentName:"ol"},(0,o.kt)("p",{parentName:"li"},"Click ",(0,o.kt)("strong",{parentName:"p"},"Run")," to launch your query. The query returns information including its duration and the number of rows inserted into the table."))),(0,o.kt)("h2",{id:"query-the-data"},"Query the data"),(0,o.kt)("p",null,"You can query the ",(0,o.kt)("inlineCode",{parentName:"p"},"wikipedia")," table after the ingestion completes.\nFor example, you can analyze the data in the table to produce a list of top channels:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-sql"},'SELECT\n  channel,\n  COUNT(*)\nFROM "wikipedia"\nGROUP BY channel\nORDER BY COUNT(*) DESC\n')),(0,o.kt)("p",null,"With the EXTERN function, you could run the same query on the external data directly without ingesting it first:"),(0,o.kt)("details",null,(0,o.kt)("summary",null,"Show the query"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-sql"},'SELECT\n  channel,\n  COUNT(*)\nFROM TABLE(\n  EXTERN(\n    \'{"type": "http", "uris": ["https://druid.apache.org/data/wikipedia.json.gz"]}\',\n    \'{"type": "json"}\',\n    \'[{"name": "added", "type": "long"}, {"name": "channel", "type": "string"}, {"name": "cityName", "type": "string"}, {"name": "comment", "type": "string"}, {"name": "commentLength", "type": "long"}, {"name": "countryIsoCode", "type": "string"}, {"name": "countryName", "type": "string"}, {"name": "deleted", "type": "long"}, {"name": "delta", "type": "long"}, {"name": "deltaBucket", "type": "string"}, {"name": "diffUrl", "type": "string"}, {"name": "flags", "type": "string"}, {"name": "isAnonymous", "type": "string"}, {"name": "isMinor", "type": "string"}, {"name": "isNew", "type": "string"}, {"name": "isRobot", "type": "string"}, {"name": "isUnpatrolled", "type": "string"}, {"name": "metroCode", "type": "string"}, {"name": "namespace", "type": "string"}, {"name": "page", "type": "string"}, {"name": "regionIsoCode", "type": "string"}, {"name": "regionName", "type": "string"}, {"name": "timestamp", "type": "string"}, {"name": "user", "type": "string"}]\'\n  )\n)\nGROUP BY channel\nORDER BY COUNT(*) DESC\n'))),(0,o.kt)("h2",{id:"further-reading"},"Further reading"),(0,o.kt)("p",null,"See the following topics to learn more:"),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("a",{parentName:"li",href:"/docs/28.0.0/multi-stage-query/"},"SQL-based ingestion overview")," to further explore SQL-based ingestion."),(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("a",{parentName:"li",href:"/docs/28.0.0/multi-stage-query/reference"},"SQL-based ingestion reference")," for reference on context parameters, functions, and error codes.")))}g.isMDXComponent=!0}}]);