"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[9828],{3905:(e,t,n)=>{n.d(t,{Zo:()=>u,kt:()=>k});var a=n(67294);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function i(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function o(e,t){if(null==e)return{};var n,a,r=function(e,t){if(null==e)return{};var n,a,r={},l=Object.keys(e);for(a=0;a<l.length;a++)n=l[a],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(a=0;a<l.length;a++)n=l[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var s=a.createContext({}),p=function(e){var t=a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},u=function(e){var t=p(e.components);return a.createElement(s.Provider,{value:t},e.children)},d="mdxType",m={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},c=a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,l=e.originalType,s=e.parentName,u=o(e,["components","mdxType","originalType","parentName"]),d=p(n),c=r,k=d["".concat(s,".").concat(c)]||d[c]||m[c]||l;return n?a.createElement(k,i(i({ref:t},u),{},{components:n})):a.createElement(k,i({ref:t},u))}));function k(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var l=n.length,i=new Array(l);i[0]=c;var o={};for(var s in t)hasOwnProperty.call(t,s)&&(o[s]=t[s]);o.originalType=e,o[d]="string"==typeof e?e:r,i[1]=o;for(var p=2;p<l;p++)i[p]=n[p];return a.createElement.apply(null,i)}return a.createElement.apply(null,n)}c.displayName="MDXCreateElement"},41514:(e,t,n)=>{n.r(t),n.d(t,{assets:()=>u,contentTitle:()=>s,default:()=>k,frontMatter:()=>o,metadata:()=>p,toc:()=>d});var a=n(87462),r=n(63366),l=(n(67294),n(3905)),i=["components"],o={id:"native-batch-input-sources",title:"Native batch input sources",sidebar_label:"Native batch: input sources"},s=void 0,p={unversionedId:"ingestion/native-batch-input-sources",id:"ingestion/native-batch-input-sources",title:"Native batch input sources",description:"\x3c!--",source:"@site/docs/latest/ingestion/native-batch-input-source.md",sourceDirName:"ingestion",slug:"/ingestion/native-batch-input-sources",permalink:"/docs/latest/ingestion/native-batch-input-sources",draft:!1,tags:[],version:"current",frontMatter:{id:"native-batch-input-sources",title:"Native batch input sources",sidebar_label:"Native batch: input sources"}},u={},d=[{value:"S3 input source",id:"s3-input-source",level:2},{value:"Google Cloud Storage input source",id:"google-cloud-storage-input-source",level:2},{value:"Azure input source",id:"azure-input-source",level:2},{value:"HDFS input source",id:"hdfs-input-source",level:2},{value:"HTTP input source",id:"http-input-source",level:2},{value:"Inline input source",id:"inline-input-source",level:2},{value:"Local input source",id:"local-input-source",level:2},{value:"Druid input source",id:"druid-input-source",level:2},{value:"SQL input source",id:"sql-input-source",level:2},{value:"Combining input source",id:"combining-input-source",level:2}],m={toc:d},c="wrapper";function k(e){var t=e.components,n=(0,r.Z)(e,i);return(0,l.kt)(c,(0,a.Z)({},m,n,{components:t,mdxType:"MDXLayout"}),(0,l.kt)("p",null,"The input source defines where your index task reads data for Apache Druid native batch ingestion. Only the native parallel task and simple task support the input source."),(0,l.kt)("p",null,"For general information on native batch indexing and parallel task indexing, see ",(0,l.kt)("a",{parentName:"p",href:"/docs/latest/ingestion/native-batch"},"Native batch ingestion"),"."),(0,l.kt)("h2",{id:"s3-input-source"},"S3 input source"),(0,l.kt)("blockquote",null,(0,l.kt)("p",{parentName:"blockquote"},"You need to include the ",(0,l.kt)("a",{parentName:"p",href:"/docs/latest/development/extensions-core/s3"},(0,l.kt)("inlineCode",{parentName:"a"},"druid-s3-extensions"))," as an extension to use the S3 input source.")),(0,l.kt)("p",null,"The S3 input source reads objects directly from S3. You can specify either:"),(0,l.kt)("ul",null,(0,l.kt)("li",{parentName:"ul"},"a list of S3 URI strings"),(0,l.kt)("li",{parentName:"ul"},"a list of S3 location prefixes that attempts to list the contents and ingest\nall objects contained within the locations.")),(0,l.kt)("p",null,"The S3 input source is splittable. Therefore, you can use it with the ",(0,l.kt)("a",{parentName:"p",href:"/docs/latest/ingestion/native-batch"},"Parallel task"),". Each worker task of ",(0,l.kt)("inlineCode",{parentName:"p"},"index_parallel")," reads one or multiple objects."),(0,l.kt)("p",null,"Sample specs:"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-json"},'...\n    "ioConfig": {\n      "type": "index_parallel",\n      "inputSource": {\n        "type": "s3",\n        "objectGlob": "**.json",\n        "uris": ["s3://foo/bar/file.json", "s3://bar/foo/file2.json"]\n      },\n      "inputFormat": {\n        "type": "json"\n      },\n      ...\n    },\n...\n')),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-json"},'...\n    "ioConfig": {\n      "type": "index_parallel",\n      "inputSource": {\n        "type": "s3",\n        "objectGlob": "**.parquet",\n        "prefixes": ["s3://foo/bar/", "s3://bar/foo/"]\n      },\n      "inputFormat": {\n        "type": "json"\n      },\n      ...\n    },\n...\n')),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-json"},'...\n    "ioConfig": {\n      "type": "index_parallel",\n      "inputSource": {\n        "type": "s3",\n        "objectGlob": "**.json",\n        "objects": [\n          { "bucket": "foo", "path": "bar/file1.json"},\n          { "bucket": "bar", "path": "foo/file2.json"}\n        ]\n      },\n      "inputFormat": {\n        "type": "json"\n      },\n      ...\n    },\n...\n')),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-json"},'...\n    "ioConfig": {\n      "type": "index_parallel",\n      "inputSource": {\n        "type": "s3",\n        "objectGlob": "**.json",\n        "uris": ["s3://foo/bar/file.json", "s3://bar/foo/file2.json"],\n        "properties": {\n          "accessKeyId": "KLJ78979SDFdS2",\n          "secretAccessKey": "KLS89s98sKJHKJKJH8721lljkd"\n        }\n      },\n      "inputFormat": {\n        "type": "json"\n      },\n      ...\n    },\n...\n')),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-json"},'...\n    "ioConfig": {\n      "type": "index_parallel",\n      "inputSource": {\n        "type": "s3",\n        "objectGlob": "**.json",\n        "uris": ["s3://foo/bar/file.json", "s3://bar/foo/file2.json"],\n        "properties": {\n          "accessKeyId": "KLJ78979SDFdS2",\n          "secretAccessKey": "KLS89s98sKJHKJKJH8721lljkd",\n          "assumeRoleArn": "arn:aws:iam::2981002874992:role/role-s3"\n        }\n      },\n      "inputFormat": {\n        "type": "json"\n      },\n      ...\n    },\n...\n')),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-json"},'...\n    "ioConfig": {\n      "type": "index_parallel",\n      "inputSource": {\n        "type": "s3",\n        "uris": ["s3://foo/bar/file.json", "s3://bar/foo/file2.json"],\n        "endpointConfig": {\n             "url" : "s3-store.aws.com",\n             "signingRegion" : "us-west-2"\n         },\n         "clientConfig": {\n             "protocol" : "http",\n             "disableChunkedEncoding" : true,\n             "enablePathStyleAccess" : true,\n             "forceGlobalBucketAccessEnabled" : false\n         },\n         "proxyConfig": {\n             "host" : "proxy-s3.aws.com",\n             "port" : 8888,\n             "username" : "admin",\n             "password" : "admin"\n         },\n\n        "properties": {\n          "accessKeyId": "KLJ78979SDFdS2",\n          "secretAccessKey": "KLS89s98sKJHKJKJH8721lljkd",\n          "assumeRoleArn": "arn:aws:iam::2981002874992:role/role-s3"\n        }\n      },\n      "inputFormat": {\n        "type": "json"\n      },\n      ...\n    },\n...\n')),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Property"),(0,l.kt)("th",{parentName:"tr",align:null},"Description"),(0,l.kt)("th",{parentName:"tr",align:null},"Default"),(0,l.kt)("th",{parentName:"tr",align:null},"Required"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"type"),(0,l.kt)("td",{parentName:"tr",align:null},"Set the value to ",(0,l.kt)("inlineCode",{parentName:"td"},"s3"),"."),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},"yes")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"uris"),(0,l.kt)("td",{parentName:"tr",align:null},"JSON array of URIs where S3 objects to be ingested are located."),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"uris")," or ",(0,l.kt)("inlineCode",{parentName:"td"},"prefixes")," or ",(0,l.kt)("inlineCode",{parentName:"td"},"objects")," must be set")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"prefixes"),(0,l.kt)("td",{parentName:"tr",align:null},"JSON array of URI prefixes for the locations of S3 objects to be ingested. Empty objects starting with one of the given prefixes will be skipped."),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"uris")," or ",(0,l.kt)("inlineCode",{parentName:"td"},"prefixes")," or ",(0,l.kt)("inlineCode",{parentName:"td"},"objects")," must be set")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"objects"),(0,l.kt)("td",{parentName:"tr",align:null},"JSON array of S3 Objects to be ingested."),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"uris")," or ",(0,l.kt)("inlineCode",{parentName:"td"},"prefixes")," or ",(0,l.kt)("inlineCode",{parentName:"td"},"objects")," must be set")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"objectGlob"),(0,l.kt)("td",{parentName:"tr",align:null},"A glob for the object part of the S3 URI. In the URI ",(0,l.kt)("inlineCode",{parentName:"td"},"s3://foo/bar/file.json"),", the glob is applied to ",(0,l.kt)("inlineCode",{parentName:"td"},"bar/file.json"),".",(0,l.kt)("br",null),(0,l.kt)("br",null),"The glob must match the entire object part, not just the filename. For example, the glob ",(0,l.kt)("inlineCode",{parentName:"td"},"*.json")," does not match ",(0,l.kt)("inlineCode",{parentName:"td"},"s3://foo/bar/file.json"),", because the object part is ",(0,l.kt)("inlineCode",{parentName:"td"},"bar/file.json"),", and the",(0,l.kt)("inlineCode",{parentName:"td"},"*")," does not match the slash. To match all objects ending in ",(0,l.kt)("inlineCode",{parentName:"td"},".json"),", use ",(0,l.kt)("inlineCode",{parentName:"td"},"**.json")," instead.",(0,l.kt)("br",null),(0,l.kt)("br",null),"For more information, refer to the documentation for ",(0,l.kt)("a",{parentName:"td",href:"https://docs.oracle.com/javase/8/docs/api/java/nio/file/FileSystem.html#getPathMatcher-java.lang.String-"},(0,l.kt)("inlineCode",{parentName:"a"},"FileSystem#getPathMatcher")),"."),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},"no")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"endpointConfig"),(0,l.kt)("td",{parentName:"tr",align:null},"Config for overriding the default S3 endpoint and signing region. This would allow ingesting data from a different S3 store. Please see ",(0,l.kt)("a",{parentName:"td",href:"/docs/latest/development/extensions-core/s3#connecting-to-s3-configuration"},"s3 config")," for more information."),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},"No (defaults will be used if not given)")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"clientConfig"),(0,l.kt)("td",{parentName:"tr",align:null},"S3 client properties for the overridden s3 endpoint. This is used in conjunction with ",(0,l.kt)("inlineCode",{parentName:"td"},"endPointConfig"),". Please see ",(0,l.kt)("a",{parentName:"td",href:"/docs/latest/development/extensions-core/s3#connecting-to-s3-configuration"},"s3 config")," for more information."),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},"No (defaults will be used if not given)")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"proxyConfig"),(0,l.kt)("td",{parentName:"tr",align:null},"Properties for specifying proxy information for the overridden s3 endpoint. This is used in conjunction with ",(0,l.kt)("inlineCode",{parentName:"td"},"clientConfig"),". Please see ",(0,l.kt)("a",{parentName:"td",href:"/docs/latest/development/extensions-core/s3#connecting-to-s3-configuration"},"s3 config")," for more information."),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},"No (defaults will be used if not given)")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"properties"),(0,l.kt)("td",{parentName:"tr",align:null},"Properties Object for overriding the default S3 configuration. See below for more information."),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},"No (defaults will be used if not given)")))),(0,l.kt)("p",null,"Note that the S3 input source will skip all empty objects only when ",(0,l.kt)("inlineCode",{parentName:"p"},"prefixes")," is specified."),(0,l.kt)("p",null,"S3 Object:"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Property"),(0,l.kt)("th",{parentName:"tr",align:null},"Description"),(0,l.kt)("th",{parentName:"tr",align:null},"Default"),(0,l.kt)("th",{parentName:"tr",align:null},"Required"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"bucket"),(0,l.kt)("td",{parentName:"tr",align:null},"Name of the S3 bucket"),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},"yes")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"path"),(0,l.kt)("td",{parentName:"tr",align:null},"The path where data is located."),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},"yes")))),(0,l.kt)("p",null,"Properties Object:"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Property"),(0,l.kt)("th",{parentName:"tr",align:null},"Description"),(0,l.kt)("th",{parentName:"tr",align:null},"Default"),(0,l.kt)("th",{parentName:"tr",align:null},"Required"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"accessKeyId"),(0,l.kt)("td",{parentName:"tr",align:null},"The ",(0,l.kt)("a",{parentName:"td",href:"/docs/latest/operations/password-provider"},"Password Provider")," or plain text string of this S3 input source access key"),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},"yes if secretAccessKey is given")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"secretAccessKey"),(0,l.kt)("td",{parentName:"tr",align:null},"The ",(0,l.kt)("a",{parentName:"td",href:"/docs/latest/operations/password-provider"},"Password Provider")," or plain text string of this S3 input source secret key"),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},"yes if accessKeyId is given")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"assumeRoleArn"),(0,l.kt)("td",{parentName:"tr",align:null},"AWS ARN of the role to assume ",(0,l.kt)("a",{parentName:"td",href:"https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html"},"see"),". ",(0,l.kt)("strong",{parentName:"td"},"assumeRoleArn")," can be used either with the ingestion spec AWS credentials or with the default S3 credentials"),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},"no")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"assumeRoleExternalId"),(0,l.kt)("td",{parentName:"tr",align:null},"A unique identifier that might be required when you assume a role in another account ",(0,l.kt)("a",{parentName:"td",href:"https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html"},"see")),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},"no")))),(0,l.kt)("blockquote",null,(0,l.kt)("p",{parentName:"blockquote"},(0,l.kt)("strong",{parentName:"p"},"Note:")," If ",(0,l.kt)("inlineCode",{parentName:"p"},"accessKeyId")," and ",(0,l.kt)("inlineCode",{parentName:"p"},"secretAccessKey")," are not given, the default ",(0,l.kt)("a",{parentName:"p",href:"/docs/latest/development/extensions-core/s3#s3-authentication-methods"},"S3 credentials provider chain")," is used.")),(0,l.kt)("h2",{id:"google-cloud-storage-input-source"},"Google Cloud Storage input source"),(0,l.kt)("blockquote",null,(0,l.kt)("p",{parentName:"blockquote"},"You need to include the ",(0,l.kt)("a",{parentName:"p",href:"/docs/latest/development/extensions-core/google"},(0,l.kt)("inlineCode",{parentName:"a"},"druid-google-extensions"))," as an extension to use the Google Cloud Storage input source.")),(0,l.kt)("p",null,"The Google Cloud Storage input source is to support reading objects directly\nfrom Google Cloud Storage. Objects can be specified as list of Google\nCloud Storage URI strings. The Google Cloud Storage input source is splittable\nand can be used by the ",(0,l.kt)("a",{parentName:"p",href:"/docs/latest/ingestion/native-batch"},"Parallel task"),", where each worker task of ",(0,l.kt)("inlineCode",{parentName:"p"},"index_parallel")," will read\none or multiple objects."),(0,l.kt)("p",null,"Sample specs:"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-json"},'...\n    "ioConfig": {\n      "type": "index_parallel",\n      "inputSource": {\n        "type": "google",\n        "objectGlob": "**.json",\n        "uris": ["gs://foo/bar/file.json", "gs://bar/foo/file2.json"]\n      },\n      "inputFormat": {\n        "type": "json"\n      },\n      ...\n    },\n...\n')),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-json"},'...\n    "ioConfig": {\n      "type": "index_parallel",\n      "inputSource": {\n        "type": "google",\n        "objectGlob": "**.parquet",\n        "prefixes": ["gs://foo/bar/", "gs://bar/foo/"]\n      },\n      "inputFormat": {\n        "type": "json"\n      },\n      ...\n    },\n...\n')),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-json"},'...\n    "ioConfig": {\n      "type": "index_parallel",\n      "inputSource": {\n        "type": "google",\n        "objectGlob": "**.json",\n        "objects": [\n          { "bucket": "foo", "path": "bar/file1.json"},\n          { "bucket": "bar", "path": "foo/file2.json"}\n        ]\n      },\n      "inputFormat": {\n        "type": "json"\n      },\n      ...\n    },\n...\n')),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Property"),(0,l.kt)("th",{parentName:"tr",align:null},"Description"),(0,l.kt)("th",{parentName:"tr",align:null},"Default"),(0,l.kt)("th",{parentName:"tr",align:null},"Required"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"type"),(0,l.kt)("td",{parentName:"tr",align:null},"Set the value to ",(0,l.kt)("inlineCode",{parentName:"td"},"google"),"."),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},"yes")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"uris"),(0,l.kt)("td",{parentName:"tr",align:null},"JSON array of URIs where Google Cloud Storage objects to be ingested are located."),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"uris")," or ",(0,l.kt)("inlineCode",{parentName:"td"},"prefixes")," or ",(0,l.kt)("inlineCode",{parentName:"td"},"objects")," must be set")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"prefixes"),(0,l.kt)("td",{parentName:"tr",align:null},"JSON array of URI prefixes for the locations of Google Cloud Storage objects to be ingested. Empty objects starting with one of the given prefixes will be skipped."),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"uris")," or ",(0,l.kt)("inlineCode",{parentName:"td"},"prefixes")," or ",(0,l.kt)("inlineCode",{parentName:"td"},"objects")," must be set")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"objects"),(0,l.kt)("td",{parentName:"tr",align:null},"JSON array of Google Cloud Storage objects to be ingested."),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"uris")," or ",(0,l.kt)("inlineCode",{parentName:"td"},"prefixes")," or ",(0,l.kt)("inlineCode",{parentName:"td"},"objects")," must be set")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"objectGlob"),(0,l.kt)("td",{parentName:"tr",align:null},"A glob for the object part of the S3 URI. In the URI ",(0,l.kt)("inlineCode",{parentName:"td"},"s3://foo/bar/file.json"),", the glob is applied to ",(0,l.kt)("inlineCode",{parentName:"td"},"bar/file.json"),".",(0,l.kt)("br",null),(0,l.kt)("br",null),"The glob must match the entire object part, not just the filename. For example, the glob ",(0,l.kt)("inlineCode",{parentName:"td"},"*.json")," does not match ",(0,l.kt)("inlineCode",{parentName:"td"},"s3://foo/bar/file.json"),", because the object part is ",(0,l.kt)("inlineCode",{parentName:"td"},"bar/file.json"),", and the",(0,l.kt)("inlineCode",{parentName:"td"},"*")," does not match the slash. To match all objects ending in ",(0,l.kt)("inlineCode",{parentName:"td"},".json"),", use ",(0,l.kt)("inlineCode",{parentName:"td"},"**.json")," instead.",(0,l.kt)("br",null),(0,l.kt)("br",null),"For more information, refer to the documentation for ",(0,l.kt)("a",{parentName:"td",href:"https://docs.oracle.com/javase/8/docs/api/java/nio/file/FileSystem.html#getPathMatcher-java.lang.String-"},(0,l.kt)("inlineCode",{parentName:"a"},"FileSystem#getPathMatcher")),"."),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},"no")))),(0,l.kt)("p",null,"Note that the Google Cloud Storage input source will skip all empty objects only when ",(0,l.kt)("inlineCode",{parentName:"p"},"prefixes")," is specified."),(0,l.kt)("p",null,"Google Cloud Storage object:"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Property"),(0,l.kt)("th",{parentName:"tr",align:null},"Description"),(0,l.kt)("th",{parentName:"tr",align:null},"Default"),(0,l.kt)("th",{parentName:"tr",align:null},"Required"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"bucket"),(0,l.kt)("td",{parentName:"tr",align:null},"Name of the Google Cloud Storage bucket"),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},"yes")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"path"),(0,l.kt)("td",{parentName:"tr",align:null},"The path where data is located."),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},"yes")))),(0,l.kt)("h2",{id:"azure-input-source"},"Azure input source"),(0,l.kt)("blockquote",null,(0,l.kt)("p",{parentName:"blockquote"},"You need to include the ",(0,l.kt)("a",{parentName:"p",href:"/docs/latest/development/extensions-core/azure"},(0,l.kt)("inlineCode",{parentName:"a"},"druid-azure-extensions"))," as an extension to use the Azure input source.")),(0,l.kt)("p",null,"The Azure input source reads objects directly from Azure Blob store or Azure Data Lake sources. You can\nspecify objects as a list of file URI strings or prefixes. You can split the Azure input source for use with ",(0,l.kt)("a",{parentName:"p",href:"/docs/latest/ingestion/native-batch"},"Parallel task")," indexing and each worker task reads one chunk of the split data."),(0,l.kt)("p",null,"Sample specs:"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-json"},'...\n    "ioConfig": {\n      "type": "index_parallel",\n      "inputSource": {\n        "type": "azure",\n        "objectGlob": "**.json",\n        "uris": ["azure://container/prefix1/file.json", "azure://container/prefix2/file2.json"]\n      },\n      "inputFormat": {\n        "type": "json"\n      },\n      ...\n    },\n...\n')),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-json"},'...\n    "ioConfig": {\n      "type": "index_parallel",\n      "inputSource": {\n        "type": "azure",\n        "objectGlob": "**.parquet",\n        "prefixes": ["azure://container/prefix1/", "azure://container/prefix2/"]\n      },\n      "inputFormat": {\n        "type": "json"\n      },\n      ...\n    },\n...\n')),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-json"},'...\n    "ioConfig": {\n      "type": "index_parallel",\n      "inputSource": {\n        "type": "azure",\n        "objectGlob": "**.json",\n        "objects": [\n          { "bucket": "container", "path": "prefix1/file1.json"},\n          { "bucket": "container", "path": "prefix2/file2.json"}\n        ]\n      },\n      "inputFormat": {\n        "type": "json"\n      },\n      ...\n    },\n...\n')),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Property"),(0,l.kt)("th",{parentName:"tr",align:null},"Description"),(0,l.kt)("th",{parentName:"tr",align:null},"Default"),(0,l.kt)("th",{parentName:"tr",align:null},"Required"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"type"),(0,l.kt)("td",{parentName:"tr",align:null},"Set the value to ",(0,l.kt)("inlineCode",{parentName:"td"},"azure"),"."),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},"yes")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"uris"),(0,l.kt)("td",{parentName:"tr",align:null},"JSON array of URIs where the Azure objects to be ingested are located, in the form ",(0,l.kt)("inlineCode",{parentName:"td"},"azure://<container>/<path-to-file>")),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"uris")," or ",(0,l.kt)("inlineCode",{parentName:"td"},"prefixes")," or ",(0,l.kt)("inlineCode",{parentName:"td"},"objects")," must be set")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"prefixes"),(0,l.kt)("td",{parentName:"tr",align:null},"JSON array of URI prefixes for the locations of Azure objects to ingest, in the form ",(0,l.kt)("inlineCode",{parentName:"td"},"azure://<container>/<prefix>"),". Empty objects starting with one of the given prefixes are skipped."),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"uris")," or ",(0,l.kt)("inlineCode",{parentName:"td"},"prefixes")," or ",(0,l.kt)("inlineCode",{parentName:"td"},"objects")," must be set")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"objects"),(0,l.kt)("td",{parentName:"tr",align:null},"JSON array of Azure objects to ingest."),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"uris")," or ",(0,l.kt)("inlineCode",{parentName:"td"},"prefixes")," or ",(0,l.kt)("inlineCode",{parentName:"td"},"objects")," must be set")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"objectGlob"),(0,l.kt)("td",{parentName:"tr",align:null},"A glob for the object part of the S3 URI. In the URI ",(0,l.kt)("inlineCode",{parentName:"td"},"s3://foo/bar/file.json"),", the glob is applied to ",(0,l.kt)("inlineCode",{parentName:"td"},"bar/file.json"),".",(0,l.kt)("br",null),(0,l.kt)("br",null),"The glob must match the entire object part, not just the filename. For example, the glob ",(0,l.kt)("inlineCode",{parentName:"td"},"*.json")," does not match ",(0,l.kt)("inlineCode",{parentName:"td"},"s3://foo/bar/file.json"),", because the object part is ",(0,l.kt)("inlineCode",{parentName:"td"},"bar/file.json"),", and the",(0,l.kt)("inlineCode",{parentName:"td"},"*")," does not match the slash. To match all objects ending in ",(0,l.kt)("inlineCode",{parentName:"td"},".json"),", use ",(0,l.kt)("inlineCode",{parentName:"td"},"**.json")," instead.",(0,l.kt)("br",null),(0,l.kt)("br",null),"For more information, refer to the documentation for ",(0,l.kt)("a",{parentName:"td",href:"https://docs.oracle.com/javase/8/docs/api/java/nio/file/FileSystem.html#getPathMatcher-java.lang.String-"},(0,l.kt)("inlineCode",{parentName:"a"},"FileSystem#getPathMatcher")),"."),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},"no")))),(0,l.kt)("p",null,"Note that the Azure input source skips all empty objects only when ",(0,l.kt)("inlineCode",{parentName:"p"},"prefixes")," is specified."),(0,l.kt)("p",null,"The ",(0,l.kt)("inlineCode",{parentName:"p"},"objects")," property is:"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Property"),(0,l.kt)("th",{parentName:"tr",align:null},"Description"),(0,l.kt)("th",{parentName:"tr",align:null},"Default"),(0,l.kt)("th",{parentName:"tr",align:null},"Required"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"bucket"),(0,l.kt)("td",{parentName:"tr",align:null},"Name of the Azure Blob Storage or Azure Data Lake container"),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},"yes")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"path"),(0,l.kt)("td",{parentName:"tr",align:null},"The path where data is located."),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},"yes")))),(0,l.kt)("h2",{id:"hdfs-input-source"},"HDFS input source"),(0,l.kt)("blockquote",null,(0,l.kt)("p",{parentName:"blockquote"},"You need to include the ",(0,l.kt)("a",{parentName:"p",href:"/docs/latest/development/extensions-core/hdfs"},(0,l.kt)("inlineCode",{parentName:"a"},"druid-hdfs-storage"))," as an extension to use the HDFS input source.")),(0,l.kt)("p",null,"The HDFS input source is to support reading files directly\nfrom HDFS storage. File paths can be specified as an HDFS URI string or a list\nof HDFS URI strings. The HDFS input source is splittable and can be used by the ",(0,l.kt)("a",{parentName:"p",href:"/docs/latest/ingestion/native-batch"},"Parallel task"),",\nwhere each worker task of ",(0,l.kt)("inlineCode",{parentName:"p"},"index_parallel")," will read one or multiple files."),(0,l.kt)("p",null,"Sample specs:"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-json"},'...\n    "ioConfig": {\n      "type": "index_parallel",\n      "inputSource": {\n        "type": "hdfs",\n        "paths": "hdfs://namenode_host/foo/bar/", "hdfs://namenode_host/bar/foo"\n      },\n      "inputFormat": {\n        "type": "json"\n      },\n      ...\n    },\n...\n')),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-json"},'...\n    "ioConfig": {\n      "type": "index_parallel",\n      "inputSource": {\n        "type": "hdfs",\n        "paths": "hdfs://namenode_host/foo/bar/", "hdfs://namenode_host/bar/foo"\n      },\n      "inputFormat": {\n        "type": "json"\n      },\n      ...\n    },\n...\n')),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-json"},'...\n    "ioConfig": {\n      "type": "index_parallel",\n      "inputSource": {\n        "type": "hdfs",\n        "paths": "hdfs://namenode_host/foo/bar/file.json", "hdfs://namenode_host/bar/foo/file2.json"\n      },\n      "inputFormat": {\n        "type": "json"\n      },\n      ...\n    },\n...\n')),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-json"},'...\n    "ioConfig": {\n      "type": "index_parallel",\n      "inputSource": {\n        "type": "hdfs",\n        "paths": ["hdfs://namenode_host/foo/bar/file.json", "hdfs://namenode_host/bar/foo/file2.json"]\n      },\n      "inputFormat": {\n        "type": "json"\n      },\n      ...\n    },\n...\n')),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Property"),(0,l.kt)("th",{parentName:"tr",align:null},"Description"),(0,l.kt)("th",{parentName:"tr",align:null},"Default"),(0,l.kt)("th",{parentName:"tr",align:null},"Required"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"type"),(0,l.kt)("td",{parentName:"tr",align:null},"Set the value to ",(0,l.kt)("inlineCode",{parentName:"td"},"hdfs"),"."),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},"yes")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"paths"),(0,l.kt)("td",{parentName:"tr",align:null},"HDFS paths. Can be either a JSON array or comma-separated string of paths. Wildcards like ",(0,l.kt)("inlineCode",{parentName:"td"},"*")," are supported in these paths. Empty files located under one of the given paths will be skipped."),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},"yes")))),(0,l.kt)("p",null,"You can also ingest from other storage using the HDFS input source if the HDFS client supports that storage.\nHowever, if you want to ingest from cloud storage, consider using the service-specific input source for your data storage.\nIf you want to use a non-hdfs protocol with the HDFS input source, include the protocol\nin ",(0,l.kt)("inlineCode",{parentName:"p"},"druid.ingestion.hdfs.allowedProtocols"),". See ",(0,l.kt)("a",{parentName:"p",href:"/docs/latest/configuration/#hdfs-input-source"},"HDFS input source security configuration")," for more details."),(0,l.kt)("h2",{id:"http-input-source"},"HTTP input source"),(0,l.kt)("p",null,"The HTTP input source is to support reading files directly from remote sites via HTTP."),(0,l.kt)("blockquote",null,(0,l.kt)("p",{parentName:"blockquote"},(0,l.kt)("strong",{parentName:"p"},"Security notes:")," Ingestion tasks run under the operating system account that runs the Druid processes, for example the Indexer, Middle Manager, and Peon. This means any user who can submit an ingestion task can specify an input source referring to any location that the Druid process can access. For example, using ",(0,l.kt)("inlineCode",{parentName:"p"},"http")," input source, users may have access to internal network servers."),(0,l.kt)("p",{parentName:"blockquote"},"The ",(0,l.kt)("inlineCode",{parentName:"p"},"http")," input source is not limited to the HTTP or HTTPS protocols. It uses the Java URI class that supports HTTP, HTTPS, FTP, file, and jar protocols by default.")),(0,l.kt)("p",null,"For more information about security best practices, see ",(0,l.kt)("a",{parentName:"p",href:"/docs/latest/operations/security-overview#best-practices"},"Security overview"),"."),(0,l.kt)("p",null,"The HTTP input source is ",(0,l.kt)("em",{parentName:"p"},"splittable")," and can be used by the ",(0,l.kt)("a",{parentName:"p",href:"/docs/latest/ingestion/native-batch"},"Parallel task"),",\nwhere each worker task of ",(0,l.kt)("inlineCode",{parentName:"p"},"index_parallel")," will read only one file. This input source does not support Split Hint Spec."),(0,l.kt)("p",null,"Sample specs:"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-json"},'...\n    "ioConfig": {\n      "type": "index_parallel",\n      "inputSource": {\n        "type": "http",\n        "uris": ["http://example.com/uri1", "http://example2.com/uri2"]\n      },\n      "inputFormat": {\n        "type": "json"\n      },\n      ...\n    },\n...\n')),(0,l.kt)("p",null,"Example with authentication fields using the DefaultPassword provider (this requires the password to be in the ingestion spec):"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-json"},'...\n    "ioConfig": {\n      "type": "index_parallel",\n      "inputSource": {\n        "type": "http",\n        "uris": ["http://example.com/uri1", "http://example2.com/uri2"],\n        "httpAuthenticationUsername": "username",\n        "httpAuthenticationPassword": "password123"\n      },\n      "inputFormat": {\n        "type": "json"\n      },\n      ...\n    },\n...\n')),(0,l.kt)("p",null,"You can also use the other existing Druid PasswordProviders. Here is an example using the EnvironmentVariablePasswordProvider:"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-json"},'...\n    "ioConfig": {\n      "type": "index_parallel",\n      "inputSource": {\n        "type": "http",\n        "uris": ["http://example.com/uri1", "http://example2.com/uri2"],\n        "httpAuthenticationUsername": "username",\n        "httpAuthenticationPassword": {\n          "type": "environment",\n          "variable": "HTTP_INPUT_SOURCE_PW"\n        }\n      },\n      "inputFormat": {\n        "type": "json"\n      },\n      ...\n    },\n...\n}\n')),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Property"),(0,l.kt)("th",{parentName:"tr",align:null},"Description"),(0,l.kt)("th",{parentName:"tr",align:null},"Default"),(0,l.kt)("th",{parentName:"tr",align:null},"Required"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"type"),(0,l.kt)("td",{parentName:"tr",align:null},"Set the value to ",(0,l.kt)("inlineCode",{parentName:"td"},"http"),"."),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},"yes")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"uris"),(0,l.kt)("td",{parentName:"tr",align:null},"URIs of the input files. See below for the protocols allowed for URIs."),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},"yes")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"httpAuthenticationUsername"),(0,l.kt)("td",{parentName:"tr",align:null},"Username to use for authentication with specified URIs. Can be optionally used if the URIs specified in the spec require a Basic Authentication Header."),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},"no")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"httpAuthenticationPassword"),(0,l.kt)("td",{parentName:"tr",align:null},"PasswordProvider to use with specified URIs. Can be optionally used if the URIs specified in the spec require a Basic Authentication Header."),(0,l.kt)("td",{parentName:"tr",align:null},"None"),(0,l.kt)("td",{parentName:"tr",align:null},"no")))),(0,l.kt)("p",null,"You can only use protocols listed in the ",(0,l.kt)("inlineCode",{parentName:"p"},"druid.ingestion.http.allowedProtocols")," property as HTTP input sources.\nThe ",(0,l.kt)("inlineCode",{parentName:"p"},"http")," and ",(0,l.kt)("inlineCode",{parentName:"p"},"https")," protocols are allowed by default. See ",(0,l.kt)("a",{parentName:"p",href:"/docs/latest/configuration/#http-input-source"},"HTTP input source security configuration")," for more details."),(0,l.kt)("h2",{id:"inline-input-source"},"Inline input source"),(0,l.kt)("p",null,"The Inline input source can be used to read the data inlined in its own spec.\nIt can be used for demos or for quickly testing out parsing and schema."),(0,l.kt)("p",null,"Sample spec:"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-json"},'...\n    "ioConfig": {\n      "type": "index_parallel",\n      "inputSource": {\n        "type": "inline",\n        "data": "0,values,formatted\\n1,as,CSV"\n      },\n      "inputFormat": {\n        "type": "csv"\n      },\n      ...\n    },\n...\n')),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Property"),(0,l.kt)("th",{parentName:"tr",align:null},"Description"),(0,l.kt)("th",{parentName:"tr",align:null},"Required"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"type"),(0,l.kt)("td",{parentName:"tr",align:null},"Set the value to ",(0,l.kt)("inlineCode",{parentName:"td"},"inline"),"."),(0,l.kt)("td",{parentName:"tr",align:null},"yes")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"data"),(0,l.kt)("td",{parentName:"tr",align:null},"Inlined data to ingest."),(0,l.kt)("td",{parentName:"tr",align:null},"yes")))),(0,l.kt)("h2",{id:"local-input-source"},"Local input source"),(0,l.kt)("p",null,"The Local input source is to support reading files directly from local storage,\nand is mainly intended for proof-of-concept testing.\nThe Local input source is ",(0,l.kt)("em",{parentName:"p"},"splittable")," and can be used by the ",(0,l.kt)("a",{parentName:"p",href:"/docs/latest/ingestion/native-batch"},"Parallel task"),",\nwhere each worker task of ",(0,l.kt)("inlineCode",{parentName:"p"},"index_parallel")," will read one or multiple files."),(0,l.kt)("p",null,"Sample spec:"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-json"},'...\n    "ioConfig": {\n      "type": "index_parallel",\n      "inputSource": {\n        "type": "local",\n        "filter" : "*.csv",\n        "baseDir": "/data/directory",\n        "files": ["/bar/foo", "/foo/bar"]\n      },\n      "inputFormat": {\n        "type": "csv"\n      },\n      ...\n    },\n...\n')),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Property"),(0,l.kt)("th",{parentName:"tr",align:null},"Description"),(0,l.kt)("th",{parentName:"tr",align:null},"Required"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"type"),(0,l.kt)("td",{parentName:"tr",align:null},"Set the value to ",(0,l.kt)("inlineCode",{parentName:"td"},"local"),"."),(0,l.kt)("td",{parentName:"tr",align:null},"yes")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"filter"),(0,l.kt)("td",{parentName:"tr",align:null},"A wildcard filter for files. See ",(0,l.kt)("a",{parentName:"td",href:"http://commons.apache.org/proper/commons-io/apidocs/org/apache/commons/io/filefilter/WildcardFileFilter"},"here")," for more information. Files matching the filter criteria are considered for ingestion. Files not matching the filter criteria are ignored."),(0,l.kt)("td",{parentName:"tr",align:null},"yes if ",(0,l.kt)("inlineCode",{parentName:"td"},"baseDir")," is specified")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"baseDir"),(0,l.kt)("td",{parentName:"tr",align:null},"Directory to search recursively for files to be ingested. Empty files under the ",(0,l.kt)("inlineCode",{parentName:"td"},"baseDir")," will be skipped."),(0,l.kt)("td",{parentName:"tr",align:null},"At least one of ",(0,l.kt)("inlineCode",{parentName:"td"},"baseDir")," or ",(0,l.kt)("inlineCode",{parentName:"td"},"files")," should be specified")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"files"),(0,l.kt)("td",{parentName:"tr",align:null},"File paths to ingest. Some files can be ignored to avoid ingesting duplicate files if they are located under the specified ",(0,l.kt)("inlineCode",{parentName:"td"},"baseDir"),". Empty files will be skipped."),(0,l.kt)("td",{parentName:"tr",align:null},"At least one of ",(0,l.kt)("inlineCode",{parentName:"td"},"baseDir")," or ",(0,l.kt)("inlineCode",{parentName:"td"},"files")," should be specified")))),(0,l.kt)("h2",{id:"druid-input-source"},"Druid input source"),(0,l.kt)("p",null,"The Druid input source is to support reading data directly from existing Druid segments,\npotentially using a new schema and changing the name, dimensions, metrics, rollup, etc. of the segment.\nThe Druid input source is ",(0,l.kt)("em",{parentName:"p"},"splittable")," and can be used by the ",(0,l.kt)("a",{parentName:"p",href:"/docs/latest/ingestion/native-batch"},"Parallel task"),".\nThis input source has a fixed input format for reading from Druid segments;\nno ",(0,l.kt)("inlineCode",{parentName:"p"},"inputFormat")," field needs to be specified in the ingestion spec when using this input source."),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Property"),(0,l.kt)("th",{parentName:"tr",align:null},"Description"),(0,l.kt)("th",{parentName:"tr",align:null},"Required"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"type"),(0,l.kt)("td",{parentName:"tr",align:null},"Set the value to ",(0,l.kt)("inlineCode",{parentName:"td"},"druid"),"."),(0,l.kt)("td",{parentName:"tr",align:null},"yes")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"dataSource"),(0,l.kt)("td",{parentName:"tr",align:null},"A String defining the Druid datasource to fetch rows from"),(0,l.kt)("td",{parentName:"tr",align:null},"yes")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"interval"),(0,l.kt)("td",{parentName:"tr",align:null},"A String representing an ISO-8601 interval, which defines the time range to fetch the data over."),(0,l.kt)("td",{parentName:"tr",align:null},"yes")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"filter"),(0,l.kt)("td",{parentName:"tr",align:null},"See ",(0,l.kt)("a",{parentName:"td",href:"/docs/latest/querying/filters"},"Filters"),". Only rows that match the filter, if specified, will be returned."),(0,l.kt)("td",{parentName:"tr",align:null},"no")))),(0,l.kt)("p",null,"The Druid input source can be used for a variety of purposes, including:"),(0,l.kt)("ul",null,(0,l.kt)("li",{parentName:"ul"},"Creating new datasources that are rolled-up copies of existing datasources."),(0,l.kt)("li",{parentName:"ul"},"Changing the ",(0,l.kt)("a",{parentName:"li",href:"/docs/latest/ingestion/partitioning"},"partitioning or sorting")," of a datasource to improve performance."),(0,l.kt)("li",{parentName:"ul"},"Updating or removing rows using a ",(0,l.kt)("a",{parentName:"li",href:"/docs/latest/ingestion/ingestion-spec#transformspec"},(0,l.kt)("inlineCode",{parentName:"a"},"transformSpec")),".")),(0,l.kt)("p",null,"When using the Druid input source, the timestamp column shows up as a numeric field named ",(0,l.kt)("inlineCode",{parentName:"p"},"__time")," set to the number\nof milliseconds since the epoch (January 1, 1970 00:00:00 UTC). It is common to use this in the timestampSpec, if you\nwant the output timestamp to be equivalent to the input timestamp. In this case, set the timestamp column to ",(0,l.kt)("inlineCode",{parentName:"p"},"__time"),"\nand the format to ",(0,l.kt)("inlineCode",{parentName:"p"},"auto")," or ",(0,l.kt)("inlineCode",{parentName:"p"},"millis"),"."),(0,l.kt)("p",null,"It is OK for the input and output datasources to be the same. In this case, newly generated data will overwrite the\nprevious data for the intervals specified in the ",(0,l.kt)("inlineCode",{parentName:"p"},"granularitySpec"),". Generally, if you are going to do this, it is a good\nidea to test out your reindexing by writing to a separate datasource before overwriting your main one. Alternatively, if\nyour goals can be satisfied by ",(0,l.kt)("a",{parentName:"p",href:"/docs/latest/data-management/compaction"},"compaction"),", consider that instead as a simpler\napproach."),(0,l.kt)("p",null,"An example task spec is shown below. It reads from a hypothetical raw datasource ",(0,l.kt)("inlineCode",{parentName:"p"},"wikipedia_raw")," and creates a new\nrolled-up datasource ",(0,l.kt)("inlineCode",{parentName:"p"},"wikipedia_rollup"),' by grouping on hour, "countryName", and "page".'),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-json"},'{\n  "type": "index_parallel",\n  "spec": {\n    "dataSchema": {\n      "dataSource": "wikipedia_rollup",\n      "timestampSpec": {\n        "column": "__time",\n        "format": "millis"\n      },\n      "dimensionsSpec": {\n        "dimensions": [\n          "countryName",\n          "page"\n        ]\n      },\n      "metricsSpec": [\n        {\n          "type": "count",\n          "name": "cnt"\n        }\n      ],\n      "granularitySpec": {\n        "type": "uniform",\n        "queryGranularity": "HOUR",\n        "segmentGranularity": "DAY",\n        "intervals": ["2016-06-27/P1D"],\n        "rollup": true\n      }\n    },\n    "ioConfig": {\n      "type": "index_parallel",\n      "inputSource": {\n        "type": "druid",\n        "dataSource": "wikipedia_raw",\n        "interval": "2016-06-27/P1D"\n      }\n    },\n    "tuningConfig": {\n      "type": "index_parallel",\n      "partitionsSpec": {\n        "type": "hashed"\n      },\n      "forceGuaranteedRollup": true,\n      "maxNumConcurrentSubTasks": 1\n    }\n  }\n}\n')),(0,l.kt)("blockquote",null,(0,l.kt)("p",{parentName:"blockquote"},"Note: Older versions (0.19 and earlier) did not respect the timestampSpec when using the Druid input source. If you\nhave ingestion specs that rely on this and cannot rewrite them, set\n",(0,l.kt)("a",{parentName:"p",href:"/docs/latest/configuration/#indexer-general-configuration"},(0,l.kt)("inlineCode",{parentName:"a"},"druid.indexer.task.ignoreTimestampSpecForDruidInputSource")),"\nto ",(0,l.kt)("inlineCode",{parentName:"p"},"true")," to enable a compatibility mode where the timestampSpec is ignored.")),(0,l.kt)("h2",{id:"sql-input-source"},"SQL input source"),(0,l.kt)("p",null,"The SQL input source is used to read data directly from RDBMS.\nThe SQL input source is ",(0,l.kt)("em",{parentName:"p"},"splittable")," and can be used by the ",(0,l.kt)("a",{parentName:"p",href:"/docs/latest/ingestion/native-batch"},"Parallel task"),", where each worker task will read from one SQL query from the list of queries.\nThis input source does not support Split Hint Spec.\nSince this input source has a fixed input format for reading events, no ",(0,l.kt)("inlineCode",{parentName:"p"},"inputFormat")," field needs to be specified in the ingestion spec when using this input source.\nPlease refer to the Recommended practices section below before using this input source."),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Property"),(0,l.kt)("th",{parentName:"tr",align:null},"Description"),(0,l.kt)("th",{parentName:"tr",align:null},"Required"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"type"),(0,l.kt)("td",{parentName:"tr",align:null},"Set the value to ",(0,l.kt)("inlineCode",{parentName:"td"},"sql"),"."),(0,l.kt)("td",{parentName:"tr",align:null},"Yes")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"database"),(0,l.kt)("td",{parentName:"tr",align:null},"Specifies the database connection details. The database type corresponds to the extension that supplies the ",(0,l.kt)("inlineCode",{parentName:"td"},"connectorConfig")," support. The specified extension must be loaded into Druid:",(0,l.kt)("br",null),(0,l.kt)("br",null),(0,l.kt)("ul",null,(0,l.kt)("li",null,(0,l.kt)("a",{parentName:"td",href:"/docs/latest/development/extensions-core/mysql"},"mysql-metadata-storage")," for ",(0,l.kt)("inlineCode",{parentName:"td"},"mysql")),(0,l.kt)("li",null," ",(0,l.kt)("a",{parentName:"td",href:"/docs/latest/development/extensions-core/postgresql"},"postgresql-metadata-storage")," extension for ",(0,l.kt)("inlineCode",{parentName:"td"},"postgresql"),".")),(0,l.kt)("br",null),(0,l.kt)("br",null),"You can selectively allow JDBC properties in ",(0,l.kt)("inlineCode",{parentName:"td"},"connectURI"),". See ",(0,l.kt)("a",{parentName:"td",href:"/docs/latest/configuration/#jdbc-connections-to-external-databases"},"JDBC connections security config")," for more details."),(0,l.kt)("td",{parentName:"tr",align:null},"Yes")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"foldCase"),(0,l.kt)("td",{parentName:"tr",align:null},"Toggle case folding of database column names. This may be enabled in cases where the database returns case insensitive column names in query results."),(0,l.kt)("td",{parentName:"tr",align:null},"No")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"sqls"),(0,l.kt)("td",{parentName:"tr",align:null},"List of SQL queries where each SQL query would retrieve the data to be indexed."),(0,l.kt)("td",{parentName:"tr",align:null},"Yes")))),(0,l.kt)("p",null,"The following is an example of an SQL input source spec:"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-json"},'...\n    "ioConfig": {\n      "type": "index_parallel",\n      "inputSource": {\n        "type": "sql",\n        "database": {\n            "type": "mysql",\n            "connectorConfig": {\n                "connectURI": "jdbc:mysql://host:port/schema",\n                "user": "user",\n                "password": "password"\n            }\n        },\n        "sqls": ["SELECT * FROM table1 WHERE timestamp BETWEEN \'2013-01-01 00:00:00\' AND \'2013-01-01 11:59:59\'", "SELECT * FROM table2 WHERE timestamp BETWEEN \'2013-01-01 00:00:00\' AND \'2013-01-01 11:59:59\'"]\n      }\n    },\n...\n')),(0,l.kt)("p",null,"The spec above will read all events from two separate SQLs for the interval ",(0,l.kt)("inlineCode",{parentName:"p"},"2013-01-01/2013-01-02"),".\nEach of the SQL queries will be run in its own sub-task and thus for the above example, there would be two sub-tasks."),(0,l.kt)("p",null,(0,l.kt)("strong",{parentName:"p"},"Recommended practices")),(0,l.kt)("p",null,"Compared to the other native batch input sources, SQL input source behaves differently in terms of reading the input data. Therefore, consider the following points before using this input source in a production environment:"),(0,l.kt)("ul",null,(0,l.kt)("li",{parentName:"ul"},(0,l.kt)("p",{parentName:"li"},"During indexing, each sub-task would execute one of the SQL queries and the results are stored locally on disk. The sub-tasks then proceed to read the data from these local input files and generate segments. Presently, there isn\u2019t any restriction on the size of the generated files and this would require the MiddleManagers or Indexers to have sufficient disk capacity based on the volume of data being indexed.")),(0,l.kt)("li",{parentName:"ul"},(0,l.kt)("p",{parentName:"li"},"Filtering the SQL queries based on the intervals specified in the ",(0,l.kt)("inlineCode",{parentName:"p"},"granularitySpec")," can avoid unwanted data being retrieved and stored locally by the indexing sub-tasks. For example, if the ",(0,l.kt)("inlineCode",{parentName:"p"},"intervals")," specified in the ",(0,l.kt)("inlineCode",{parentName:"p"},"granularitySpec")," is ",(0,l.kt)("inlineCode",{parentName:"p"},'["2013-01-01/2013-01-02"]')," and the SQL query is ",(0,l.kt)("inlineCode",{parentName:"p"},"SELECT * FROM table1"),", ",(0,l.kt)("inlineCode",{parentName:"p"},"SqlInputSource")," will read all the data for ",(0,l.kt)("inlineCode",{parentName:"p"},"table1")," based on the query, even though only data between the intervals specified will be indexed into Druid.")),(0,l.kt)("li",{parentName:"ul"},(0,l.kt)("p",{parentName:"li"},"Pagination may be used on the SQL queries to ensure that each query pulls a similar amount of data, thereby improving the efficiency of the sub-tasks.")),(0,l.kt)("li",{parentName:"ul"},(0,l.kt)("p",{parentName:"li"},"Similar to file-based input formats, any updates to existing data will replace the data in segments specific to the intervals specified in the ",(0,l.kt)("inlineCode",{parentName:"p"},"granularitySpec"),"."))),(0,l.kt)("h2",{id:"combining-input-source"},"Combining input source"),(0,l.kt)("p",null,"The Combining input source lets you read data from multiple input sources.\nIt identifies the splits from delegate input sources and uses a worker task to process each split.\nUse the Combining input source only if all the delegates are splittable and can be used by the ",(0,l.kt)("a",{parentName:"p",href:"/docs/latest/ingestion/native-batch"},"Parallel task"),"."),(0,l.kt)("p",null,"Similar to other input sources, the Combining input source supports a single ",(0,l.kt)("inlineCode",{parentName:"p"},"inputFormat"),".\nDelegate input sources that require an ",(0,l.kt)("inlineCode",{parentName:"p"},"inputFormat")," must have the same format for input data.\nIf you include the ",(0,l.kt)("a",{parentName:"p",href:"#druid-input-source"},"Druid input source"),", the timestamp column is stored in the ",(0,l.kt)("inlineCode",{parentName:"p"},"__time")," field.\nTo correctly combine the data from the Druid input source with another source, ensure that other delegate input sources also store the timestamp column in ",(0,l.kt)("inlineCode",{parentName:"p"},"__time"),"."),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Property"),(0,l.kt)("th",{parentName:"tr",align:null},"Description"),(0,l.kt)("th",{parentName:"tr",align:null},"Required"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"type"),(0,l.kt)("td",{parentName:"tr",align:null},"Set the value to ",(0,l.kt)("inlineCode",{parentName:"td"},"combining"),"."),(0,l.kt)("td",{parentName:"tr",align:null},"Yes")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"delegates"),(0,l.kt)("td",{parentName:"tr",align:null},"List of splittable input sources to read data from."),(0,l.kt)("td",{parentName:"tr",align:null},"Yes")))),(0,l.kt)("p",null,"The following is an example of a Combining input source spec:"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-json"},'...\n    "ioConfig": {\n      "type": "index_parallel",\n      "inputSource": {\n        "type": "combining",\n        "delegates" : [\n         {\n          "type": "local",\n          "filter" : "*.csv",\n          "baseDir": "/data/directory",\n          "files": ["/bar/foo", "/foo/bar"]\n         },\n         {\n          "type": "druid",\n          "dataSource": "wikipedia",\n          "interval": "2013-01-01/2013-01-02"\n         }\n        ]\n      },\n      "inputFormat": {\n        "type": "csv"\n      },\n      ...\n    },\n...\n')),(0,l.kt)("p",null,"The ",(0,l.kt)("a",{parentName:"p",href:"/docs/latest/ingestion/native-batch#partitionsspec"},"secondary partitioning method")," determines the requisite number of concurrent worker tasks that run in parallel to complete ingestion with the Combining input source.\nSet this value in ",(0,l.kt)("inlineCode",{parentName:"p"},"maxNumConcurrentSubTasks")," in ",(0,l.kt)("inlineCode",{parentName:"p"},"tuningConfig")," based on the secondary partitioning method:"),(0,l.kt)("ul",null,(0,l.kt)("li",{parentName:"ul"},(0,l.kt)("inlineCode",{parentName:"li"},"range")," or ",(0,l.kt)("inlineCode",{parentName:"li"},"single_dim")," partitioning: greater than or equal to 1"),(0,l.kt)("li",{parentName:"ul"},(0,l.kt)("inlineCode",{parentName:"li"},"hashed")," or ",(0,l.kt)("inlineCode",{parentName:"li"},"dynamic")," partitioning: greater than or equal to 2")),(0,l.kt)("p",null,"For more information on the ",(0,l.kt)("inlineCode",{parentName:"p"},"maxNumConcurrentSubTasks")," field, see ",(0,l.kt)("a",{parentName:"p",href:"/docs/latest/ingestion/native-batch#implementation-considerations"},"Implementation considerations"),"."))}k.isMDXComponent=!0}}]);