"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[8857],{15680:(e,t,a)=>{a.d(t,{xA:()=>d,yg:()=>u});var n=a(96540);function r(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function i(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function l(e){for(var t=1;t<arguments.length;t++){var a=null!=arguments[t]?arguments[t]:{};t%2?i(Object(a),!0).forEach((function(t){r(e,t,a[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(a)):i(Object(a)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(a,t))}))}return e}function o(e,t){if(null==e)return{};var a,n,r=function(e,t){if(null==e)return{};var a,n,r={},i=Object.keys(e);for(n=0;n<i.length;n++)a=i[n],t.indexOf(a)>=0||(r[a]=e[a]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(n=0;n<i.length;n++)a=i[n],t.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(r[a]=e[a])}return r}var p=n.createContext({}),s=function(e){var t=n.useContext(p),a=t;return e&&(a="function"==typeof e?e(t):l(l({},t),e)),a},d=function(e){var t=s(e.components);return n.createElement(p.Provider,{value:t},e.children)},g="mdxType",m={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},y=n.forwardRef((function(e,t){var a=e.components,r=e.mdxType,i=e.originalType,p=e.parentName,d=o(e,["components","mdxType","originalType","parentName"]),g=s(a),y=r,u=g["".concat(p,".").concat(y)]||g[y]||m[y]||i;return a?n.createElement(u,l(l({ref:t},d),{},{components:a})):n.createElement(u,l({ref:t},d))}));function u(e,t){var a=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=a.length,l=new Array(i);l[0]=y;var o={};for(var p in t)hasOwnProperty.call(t,p)&&(o[p]=t[p]);o.originalType=e,o[g]="string"==typeof e?e:r,l[1]=o;for(var s=2;s<i;s++)l[s]=a[s];return n.createElement.apply(null,l)}return n.createElement.apply(null,a)}y.displayName="MDXCreateElement"},4632:(e,t,a)=>{a.r(t),a.d(t,{assets:()=>d,contentTitle:()=>p,default:()=>u,frontMatter:()=>o,metadata:()=>s,toc:()=>g});var n=a(58168),r=a(98587),i=(a(96540),a(15680)),l=["components"],o={id:"data-formats",title:"Source input formats",sidebar_label:"Source input formats"},p=void 0,s={unversionedId:"ingestion/data-formats",id:"ingestion/data-formats",title:"Source input formats",description:"\x3c!--",source:"@site/docs/latest/ingestion/data-formats.md",sourceDirName:"ingestion",slug:"/ingestion/data-formats",permalink:"/docs/latest/ingestion/data-formats",draft:!1,tags:[],version:"current",frontMatter:{id:"data-formats",title:"Source input formats",sidebar_label:"Source input formats"},sidebar:"docs",previous:{title:"Overview",permalink:"/docs/latest/ingestion/"},next:{title:"Input sources",permalink:"/docs/latest/ingestion/input-sources"}},d={},g=[{value:"Formatting data",id:"formatting-data",level:2},{value:"Custom formats",id:"custom-formats",level:2},{value:"Input format",id:"input-format",level:2},{value:"JSON",id:"json",level:3},{value:"CSV",id:"csv",level:3},{value:"TSV (Delimited)",id:"tsv-delimited",level:3},{value:"ORC",id:"orc",level:3},{value:"Parquet",id:"parquet",level:3},{value:"Avro Stream",id:"avro-stream",level:3},{value:"Avro Bytes Decoder",id:"avro-bytes-decoder",level:5},{value:"Inline Schema Based Avro Bytes Decoder",id:"inline-schema-based-avro-bytes-decoder",level:6},{value:"Multiple Inline Schemas Based Avro Bytes Decoder",id:"multiple-inline-schemas-based-avro-bytes-decoder",level:6},{value:"SchemaRepo Based Avro Bytes Decoder",id:"schemarepo-based-avro-bytes-decoder",level:5},{value:"Avro-1124 Subject And Id Converter",id:"avro-1124-subject-and-id-converter",level:6},{value:"Avro-1124 Schema Repository",id:"avro-1124-schema-repository",level:6},{value:"Confluent Schema Registry-based Avro Bytes Decoder",id:"confluent-schema-registry-based-avro-bytes-decoder",level:6},{value:"Parse exceptions",id:"parse-exceptions",level:6},{value:"Avro OCF",id:"avro-ocf",level:3},{value:"Protobuf",id:"protobuf",level:3},{value:"Kafka",id:"kafka",level:3},{value:"Header format",id:"header-format",level:4},{value:"Example",id:"example",level:4},{value:"FlattenSpec",id:"flattenspec",level:2},{value:"Field flattening specifications",id:"field-flattening-specifications",level:3},{value:"Notes on flattening",id:"notes-on-flattening",level:3},{value:"Parser",id:"parser",level:2},{value:"String Parser",id:"string-parser",level:3},{value:"Avro Hadoop Parser",id:"avro-hadoop-parser",level:3},{value:"ORC Hadoop Parser",id:"orc-hadoop-parser",level:3},{value:"Hadoop job properties",id:"hadoop-job-properties",level:4},{value:"Examples",id:"examples",level:4},{value:"<code>orc</code> parser, <code>orc</code> parseSpec, auto field discovery, flatten expressions",id:"orc-parser-orc-parsespec-auto-field-discovery-flatten-expressions",level:5},{value:"<code>orc</code> parser, <code>orc</code> parseSpec, field discovery with no flattenSpec or dimensionSpec",id:"orc-parser-orc-parsespec-field-discovery-with-no-flattenspec-or-dimensionspec",level:5},{value:"<code>orc</code> parser, <code>orc</code> parseSpec, no autodiscovery",id:"orc-parser-orc-parsespec-no-autodiscovery",level:5},{value:"<code>orc</code> parser, <code>timeAndDims</code> parseSpec",id:"orc-parser-timeanddims-parsespec",level:5},{value:"Parquet Hadoop Parser",id:"parquet-hadoop-parser",level:3},{value:"Parquet Hadoop Parser vs Parquet Avro Hadoop Parser",id:"parquet-hadoop-parser-vs-parquet-avro-hadoop-parser",level:4},{value:"Examples",id:"examples-1",level:4},{value:"<code>parquet</code> parser, <code>parquet</code> parseSpec",id:"parquet-parser-parquet-parsespec",level:5},{value:"<code>parquet</code> parser, <code>timeAndDims</code> parseSpec",id:"parquet-parser-timeanddims-parsespec",level:5},{value:"Parquet Avro Hadoop Parser",id:"parquet-avro-hadoop-parser",level:3},{value:"Example",id:"example-1",level:4},{value:"Avro Stream Parser",id:"avro-stream-parser",level:3},{value:"Protobuf Parser",id:"protobuf-parser",level:3},{value:"Protobuf Bytes Decoder",id:"protobuf-bytes-decoder",level:4},{value:"File-based Protobuf Bytes Decoder",id:"file-based-protobuf-bytes-decoder",level:5},{value:"Inline Descriptor Protobuf Bytes Decoder",id:"inline-descriptor-protobuf-bytes-decoder",level:4},{value:"Confluent Schema Registry-based Protobuf Bytes Decoder",id:"confluent-schema-registry-based-protobuf-bytes-decoder",level:5},{value:"ParseSpec",id:"parsespec",level:2},{value:"JSON ParseSpec",id:"json-parsespec",level:3},{value:"JSON Lowercase ParseSpec",id:"json-lowercase-parsespec",level:3},{value:"CSV ParseSpec",id:"csv-parsespec",level:3},{value:"CSV Index Tasks",id:"csv-index-tasks",level:4},{value:"Other CSV Ingestion Tasks",id:"other-csv-ingestion-tasks",level:4},{value:"TSV / Delimited ParseSpec",id:"tsv--delimited-parsespec",level:3},{value:"TSV (Delimited) Index Tasks",id:"tsv-delimited-index-tasks",level:4},{value:"Other TSV (Delimited) Ingestion Tasks",id:"other-tsv-delimited-ingestion-tasks",level:4},{value:"Regex ParseSpec",id:"regex-parsespec",level:3},{value:"JavaScript ParseSpec",id:"javascript-parsespec",level:3},{value:"TimeAndDims ParseSpec",id:"timeanddims-parsespec",level:3},{value:"Orc ParseSpec",id:"orc-parsespec",level:3},{value:"Parquet ParseSpec",id:"parquet-parsespec",level:3}],m={toc:g},y="wrapper";function u(e){var t=e.components,a=(0,r.A)(e,l);return(0,i.yg)(y,(0,n.A)({},m,a,{components:t,mdxType:"MDXLayout"}),(0,i.yg)("p",null,"Apache Druid can ingest denormalized data in JSON, CSV, or a delimited form such as TSV, or any custom format. While most examples in the documentation use data in JSON format, it is not difficult to configure Druid to ingest any other delimited data.\nWe welcome any contributions to new formats."),(0,i.yg)("p",null,"This page lists all default and core extension data formats supported by Druid.\nFor additional data formats supported with community extensions,\nplease see our ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/configuration/extensions#community-extensions"},"community extensions list"),"."),(0,i.yg)("h2",{id:"formatting-data"},"Formatting data"),(0,i.yg)("p",null,"The following samples show data formats that are natively supported in Druid:"),(0,i.yg)("p",null,(0,i.yg)("em",{parentName:"p"},"JSON")),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'{"timestamp": "2013-08-31T01:02:33Z", "page": "Gypsy Danger", "language" : "en", "user" : "nuclear", "unpatrolled" : "true", "newPage" : "true", "robot": "false", "anonymous": "false", "namespace":"article", "continent":"North America", "country":"United States", "region":"Bay Area", "city":"San Francisco", "added": 57, "deleted": 200, "delta": -143}\n{"timestamp": "2013-08-31T03:32:45Z", "page": "Striker Eureka", "language" : "en", "user" : "speed", "unpatrolled" : "false", "newPage" : "true", "robot": "true", "anonymous": "false", "namespace":"wikipedia", "continent":"Australia", "country":"Australia", "region":"Cantebury", "city":"Syndey", "added": 459, "deleted": 129, "delta": 330}\n{"timestamp": "2013-08-31T07:11:21Z", "page": "Cherno Alpha", "language" : "ru", "user" : "masterYi", "unpatrolled" : "false", "newPage" : "true", "robot": "true", "anonymous": "false", "namespace":"article", "continent":"Asia", "country":"Russia", "region":"Oblast", "city":"Moscow", "added": 123, "deleted": 12, "delta": 111}\n{"timestamp": "2013-08-31T11:58:39Z", "page": "Crimson Typhoon", "language" : "zh", "user" : "triplets", "unpatrolled" : "true", "newPage" : "false", "robot": "true", "anonymous": "false", "namespace":"wikipedia", "continent":"Asia", "country":"China", "region":"Shanxi", "city":"Taiyuan", "added": 905, "deleted": 5, "delta": 900}\n{"timestamp": "2013-08-31T12:41:27Z", "page": "Coyote Tango", "language" : "ja", "user" : "cancer", "unpatrolled" : "true", "newPage" : "false", "robot": "true", "anonymous": "false", "namespace":"wikipedia", "continent":"Asia", "country":"Japan", "region":"Kanto", "city":"Tokyo", "added": 1, "deleted": 10, "delta": -9}\n')),(0,i.yg)("p",null,(0,i.yg)("em",{parentName:"p"},"CSV")),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre"},'2013-08-31T01:02:33Z,"Gypsy Danger","en","nuclear","true","true","false","false","article","North America","United States","Bay Area","San Francisco",57,200,-143\n2013-08-31T03:32:45Z,"Striker Eureka","en","speed","false","true","true","false","wikipedia","Australia","Australia","Cantebury","Syndey",459,129,330\n2013-08-31T07:11:21Z,"Cherno Alpha","ru","masterYi","false","true","true","false","article","Asia","Russia","Oblast","Moscow",123,12,111\n2013-08-31T11:58:39Z,"Crimson Typhoon","zh","triplets","true","false","true","false","wikipedia","Asia","China","Shanxi","Taiyuan",905,5,900\n2013-08-31T12:41:27Z,"Coyote Tango","ja","cancer","true","false","true","false","wikipedia","Asia","Japan","Kanto","Tokyo",1,10,-9\n')),(0,i.yg)("p",null,(0,i.yg)("em",{parentName:"p"},"TSV (Delimited)")),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre"},'2013-08-31T01:02:33Z  "Gypsy Danger"  "en"  "nuclear" "true"  "true"  "false" "false" "article" "North America" "United States" "Bay Area"  "San Francisco" 57  200 -143\n2013-08-31T03:32:45Z  "Striker Eureka"  "en"  "speed" "false" "true"  "true"  "false" "wikipedia" "Australia" "Australia" "Cantebury" "Syndey"  459 129 330\n2013-08-31T07:11:21Z  "Cherno Alpha"  "ru"  "masterYi"  "false" "true"  "true"  "false" "article" "Asia"  "Russia"  "Oblast"  "Moscow"  123 12  111\n2013-08-31T11:58:39Z  "Crimson Typhoon" "zh"  "triplets"  "true"  "false" "true"  "false" "wikipedia" "Asia"  "China" "Shanxi"  "Taiyuan" 905 5 900\n2013-08-31T12:41:27Z  "Coyote Tango"  "ja"  "cancer"  "true"  "false" "true"  "false" "wikipedia" "Asia"  "Japan" "Kanto" "Tokyo" 1 10  -9\n')),(0,i.yg)("p",null,"Note that the CSV and TSV data do not contain column heads. This becomes important when you specify the data for ingesting."),(0,i.yg)("p",null,"Besides text formats, Druid also supports binary formats such as ",(0,i.yg)("a",{parentName:"p",href:"#orc"},"Orc")," and ",(0,i.yg)("a",{parentName:"p",href:"#parquet"},"Parquet")," formats."),(0,i.yg)("h2",{id:"custom-formats"},"Custom formats"),(0,i.yg)("p",null,"Druid supports custom data formats and can use the Regex parser or the JavaScript parsers to parse these formats. Using any of these parsers for\nparsing data is less efficient than writing a native Java parser or using an external stream processor. We welcome contributions of new parsers."),(0,i.yg)("h2",{id:"input-format"},"Input format"),(0,i.yg)("p",null,"You can use the ",(0,i.yg)("inlineCode",{parentName:"p"},"inputFormat")," field to specify the data format for your input data."),(0,i.yg)("admonition",{type:"info"},(0,i.yg)("p",{parentName:"admonition"}," ",(0,i.yg)("inlineCode",{parentName:"p"},"inputFormat")," doesn't support all data formats or ingestion methods supported by Druid.")),(0,i.yg)("p",null,"Especially if you want to use the Hadoop ingestion, you still need to use the ",(0,i.yg)("a",{parentName:"p",href:"#parser"},"Parser"),".\nIf your data is formatted in some format not listed in this section, please consider using the Parser instead."),(0,i.yg)("p",null,"All forms of Druid ingestion require some form of schema object. The format of the data to be ingested is specified using the ",(0,i.yg)("inlineCode",{parentName:"p"},"inputFormat")," entry in your ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/ingestion/ingestion-spec#ioconfig"},(0,i.yg)("inlineCode",{parentName:"a"},"ioConfig")),"."),(0,i.yg)("h3",{id:"json"},"JSON"),(0,i.yg)("p",null,"Configure the JSON ",(0,i.yg)("inlineCode",{parentName:"p"},"inputFormat")," to load JSON data as follows:"),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"type"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"Set value to ",(0,i.yg)("inlineCode",{parentName:"td"},"json"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"flattenSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies flattening configuration for nested JSON data. See ",(0,i.yg)("a",{parentName:"td",href:"#flattenspec"},(0,i.yg)("inlineCode",{parentName:"a"},"flattenSpec"))," for more info."),(0,i.yg)("td",{parentName:"tr",align:null},"no")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"featureSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("a",{parentName:"td",href:"https://github.com/FasterXML/jackson-core/wiki/JsonParser-Features"},"JSON parser features"),' supported by Jackson, a JSON processor for Java. The features control parsing of the input JSON data. To enable a feature, map the feature name to a Boolean value of "true". For example: ',(0,i.yg)("inlineCode",{parentName:"td"},'"featureSpec": {"ALLOW_SINGLE_QUOTES": true, "ALLOW_UNQUOTED_FIELD_NAMES": true}')),(0,i.yg)("td",{parentName:"tr",align:null},"no")))),(0,i.yg)("p",null,"The following properties are specialized properties that only apply when the JSON ",(0,i.yg)("inlineCode",{parentName:"p"},"inputFormat")," is used in streaming ingestion, and they are related to how parsing exceptions are handled. In streaming ingestion, multi-line JSON events can be ingested (i.e. where a single JSON event spans multiple lines). However, if a parsing exception occurs, all JSON events that are present in the same streaming record will be discarded."),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"assumeNewlineDelimited"),(0,i.yg)("td",{parentName:"tr",align:null},"Boolean"),(0,i.yg)("td",{parentName:"tr",align:null},"If the input is known to be newline delimited JSON (each individual JSON event is contained in a single line, separated by newlines), setting this option to true allows for more flexible parsing exception handling. Only the lines with invalid JSON syntax will be discarded, while lines containing valid JSON events will still be ingested."),(0,i.yg)("td",{parentName:"tr",align:null},"no (Default false)")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"useJsonNodeReader"),(0,i.yg)("td",{parentName:"tr",align:null},"Boolean"),(0,i.yg)("td",{parentName:"tr",align:null},"When ingesting multi-line JSON events, enabling this option will enable the use of a JSON parser which will retain any valid JSON events encountered within a streaming record prior to when a parsing exception occurred."),(0,i.yg)("td",{parentName:"tr",align:null},"no (Default false)")))),(0,i.yg)("p",null,"For example:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'"ioConfig": {\n  "inputFormat": {\n    "type": "json"\n  },\n  ...\n}\n')),(0,i.yg)("h3",{id:"csv"},"CSV"),(0,i.yg)("p",null,"Configure the CSV ",(0,i.yg)("inlineCode",{parentName:"p"},"inputFormat")," to load CSV data as follows:"),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"type"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"Set value to ",(0,i.yg)("inlineCode",{parentName:"td"},"csv"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"listDelimiter"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"A custom delimiter for multi-value dimensions."),(0,i.yg)("td",{parentName:"tr",align:null},"no (default = ctrl+A)")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"columns"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON array"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the columns of the data. The columns should be in the same order with the columns of your data."),(0,i.yg)("td",{parentName:"tr",align:null},"yes if ",(0,i.yg)("inlineCode",{parentName:"td"},"findColumnsFromHeader")," is false or missing")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"findColumnsFromHeader"),(0,i.yg)("td",{parentName:"tr",align:null},"Boolean"),(0,i.yg)("td",{parentName:"tr",align:null},"If this is set, the task will find the column names from the header row. Note that ",(0,i.yg)("inlineCode",{parentName:"td"},"skipHeaderRows")," will be applied before finding column names from the header. For example, if you set ",(0,i.yg)("inlineCode",{parentName:"td"},"skipHeaderRows")," to 2 and ",(0,i.yg)("inlineCode",{parentName:"td"},"findColumnsFromHeader")," to true, the task will skip the first two lines and then extract column information from the third line. ",(0,i.yg)("inlineCode",{parentName:"td"},"columns")," will be ignored if this is set to true."),(0,i.yg)("td",{parentName:"tr",align:null},"no (default = false if ",(0,i.yg)("inlineCode",{parentName:"td"},"columns")," is set; otherwise null)")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"skipHeaderRows"),(0,i.yg)("td",{parentName:"tr",align:null},"Integer"),(0,i.yg)("td",{parentName:"tr",align:null},"If this is set, the task will skip the first ",(0,i.yg)("inlineCode",{parentName:"td"},"skipHeaderRows")," rows."),(0,i.yg)("td",{parentName:"tr",align:null},"no (default = 0)")))),(0,i.yg)("p",null,"For example:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'"ioConfig": {\n  "inputFormat": {\n    "type": "csv",\n    "columns" : ["timestamp","page","language","user","unpatrolled","newPage","robot","anonymous","namespace","continent","country","region","city","added","deleted","delta"]\n  },\n  ...\n}\n')),(0,i.yg)("h3",{id:"tsv-delimited"},"TSV (Delimited)"),(0,i.yg)("p",null,"Configure the TSV ",(0,i.yg)("inlineCode",{parentName:"p"},"inputFormat")," to load TSV data as follows:"),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"type"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"Set value to ",(0,i.yg)("inlineCode",{parentName:"td"},"tsv"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"delimiter"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"A custom delimiter for data values."),(0,i.yg)("td",{parentName:"tr",align:null},"no (default = ",(0,i.yg)("inlineCode",{parentName:"td"},"\\t"),")")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"listDelimiter"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"A custom delimiter for multi-value dimensions."),(0,i.yg)("td",{parentName:"tr",align:null},"no (default = ctrl+A)")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"columns"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON array"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the columns of the data. The columns should be in the same order with the columns of your data."),(0,i.yg)("td",{parentName:"tr",align:null},"yes if ",(0,i.yg)("inlineCode",{parentName:"td"},"findColumnsFromHeader")," is false or missing")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"findColumnsFromHeader"),(0,i.yg)("td",{parentName:"tr",align:null},"Boolean"),(0,i.yg)("td",{parentName:"tr",align:null},"If this is set, the task will find the column names from the header row. Note that ",(0,i.yg)("inlineCode",{parentName:"td"},"skipHeaderRows")," will be applied before finding column names from the header. For example, if you set ",(0,i.yg)("inlineCode",{parentName:"td"},"skipHeaderRows")," to 2 and ",(0,i.yg)("inlineCode",{parentName:"td"},"findColumnsFromHeader")," to true, the task will skip the first two lines and then extract column information from the third line. ",(0,i.yg)("inlineCode",{parentName:"td"},"columns")," will be ignored if this is set to true."),(0,i.yg)("td",{parentName:"tr",align:null},"no (default = false if ",(0,i.yg)("inlineCode",{parentName:"td"},"columns")," is set; otherwise null)")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"skipHeaderRows"),(0,i.yg)("td",{parentName:"tr",align:null},"Integer"),(0,i.yg)("td",{parentName:"tr",align:null},"If this is set, the task will skip the first ",(0,i.yg)("inlineCode",{parentName:"td"},"skipHeaderRows")," rows."),(0,i.yg)("td",{parentName:"tr",align:null},"no (default = 0)")))),(0,i.yg)("p",null,"Be sure to change the ",(0,i.yg)("inlineCode",{parentName:"p"},"delimiter")," to the appropriate delimiter for your data. Like CSV, you must specify the columns and which subset of the columns you want indexed."),(0,i.yg)("p",null,"For example:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'"ioConfig": {\n  "inputFormat": {\n    "type": "tsv",\n    "columns" : ["timestamp","page","language","user","unpatrolled","newPage","robot","anonymous","namespace","continent","country","region","city","added","deleted","delta"],\n    "delimiter":"|"\n  },\n  ...\n}\n')),(0,i.yg)("h3",{id:"orc"},"ORC"),(0,i.yg)("p",null,"To use the ORC input format, load the Druid Orc extension ( ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/development/extensions-core/orc"},(0,i.yg)("inlineCode",{parentName:"a"},"druid-orc-extensions")),")."),(0,i.yg)("admonition",{type:"info"},(0,i.yg)("p",{parentName:"admonition"}," To upgrade from versions earlier than 0.15.0 to 0.15.0 or new, read ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/development/extensions-core/orc#migration-from-contrib-extension"},"Migration from 'contrib' extension"),".")),(0,i.yg)("p",null,"Configure the ORC ",(0,i.yg)("inlineCode",{parentName:"p"},"inputFormat")," to load ORC data as follows:"),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"type"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"Set value to ",(0,i.yg)("inlineCode",{parentName:"td"},"orc"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"flattenSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies flattening configuration for nested ORC data. Only 'path' expressions are supported ('jq' and 'tree' are unavailable). See ",(0,i.yg)("a",{parentName:"td",href:"#flattenspec"},(0,i.yg)("inlineCode",{parentName:"a"},"flattenSpec"))," for more info."),(0,i.yg)("td",{parentName:"tr",align:null},"no")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"binaryAsString"),(0,i.yg)("td",{parentName:"tr",align:null},"Boolean"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies if the binary orc column which is not logically marked as a string should be treated as a UTF-8 encoded string."),(0,i.yg)("td",{parentName:"tr",align:null},"no (default = false)")))),(0,i.yg)("p",null,"For example:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'"ioConfig": {\n  "inputFormat": {\n    "type": "orc",\n    "flattenSpec": {\n      "useFieldDiscovery": true,\n      "fields": [\n        {\n          "type": "path",\n          "name": "nested",\n          "expr": "$.path.to.nested"\n        }\n      ]\n    },\n    "binaryAsString": false\n  },\n  ...\n}\n')),(0,i.yg)("h3",{id:"parquet"},"Parquet"),(0,i.yg)("p",null,"To use the Parquet input format load the Druid Parquet extension (",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/development/extensions-core/parquet"},(0,i.yg)("inlineCode",{parentName:"a"},"druid-parquet-extensions")),")."),(0,i.yg)("p",null,"Configure the Parquet ",(0,i.yg)("inlineCode",{parentName:"p"},"inputFormat")," to load Parquet data as follows:"),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"type")),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"Set value to ",(0,i.yg)("inlineCode",{parentName:"td"},"parquet"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"flattenSpec")),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Define a ",(0,i.yg)("a",{parentName:"td",href:"#flattenspec"},(0,i.yg)("inlineCode",{parentName:"a"},"flattenSpec"))," to extract nested values from a Parquet file. Only 'path' expressions are supported ('jq' and 'tree' are unavailable)."),(0,i.yg)("td",{parentName:"tr",align:null},"no (default will auto-discover 'root' level properties)")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"binaryAsString")),(0,i.yg)("td",{parentName:"tr",align:null},"Boolean"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies if the bytes parquet column which is not logically marked as a string or enum type should be treated as a UTF-8 encoded string."),(0,i.yg)("td",{parentName:"tr",align:null},"no (default = false)")))),(0,i.yg)("p",null,"For example:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'"ioConfig": {\n  "inputFormat": {\n    "type": "parquet",\n    "flattenSpec": {\n      "useFieldDiscovery": true,\n      "fields": [\n        {\n          "type": "path",\n          "name": "nested",\n          "expr": "$.path.to.nested"\n        }\n      ]\n    },\n    "binaryAsString": false\n  },\n  ...\n}\n')),(0,i.yg)("h3",{id:"avro-stream"},"Avro Stream"),(0,i.yg)("p",null,"To use the Avro Stream input format load the Druid Avro extension (",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/development/extensions-core/avro"},(0,i.yg)("inlineCode",{parentName:"a"},"druid-avro-extensions")),")."),(0,i.yg)("p",null,"For more information on how Druid handles Avro types, see ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/development/extensions-core/avro#avro-types"},"Avro Types")," section for"),(0,i.yg)("p",null,"Configure the Avro ",(0,i.yg)("inlineCode",{parentName:"p"},"inputFormat")," to load Avro data as follows:"),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"type"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"Set value to ",(0,i.yg)("inlineCode",{parentName:"td"},"avro_stream"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"flattenSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Define a ",(0,i.yg)("a",{parentName:"td",href:"#flattenspec"},(0,i.yg)("inlineCode",{parentName:"a"},"flattenSpec"))," to extract nested values from a Avro record. Only 'path' expressions are supported ('jq' is unavailable)."),(0,i.yg)("td",{parentName:"tr",align:null},"no (default will auto-discover 'root' level properties)")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"avroBytesDecoder")),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies how to decode bytes to Avro record."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"binaryAsString"),(0,i.yg)("td",{parentName:"tr",align:null},"Boolean"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies if the bytes Avro column which is not logically marked as a string or enum type should be treated as a UTF-8 encoded string."),(0,i.yg)("td",{parentName:"tr",align:null},"no (default = false)")))),(0,i.yg)("p",null,"For example:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'"ioConfig": {\n  "inputFormat": {\n    "type": "avro_stream",\n    "avroBytesDecoder": {\n      "type": "schema_inline",\n      "schema": {\n        //your schema goes here, for example\n        "namespace": "org.apache.druid.data",\n        "name": "User",\n        "type": "record",\n        "fields": [\n          { "name": "FullName", "type": "string" },\n          { "name": "Country", "type": "string" }\n        ]\n      }\n    },\n    "flattenSpec": {\n      "useFieldDiscovery": true,\n      "fields": [\n        {\n          "type": "path",\n          "name": "someRecord_subInt",\n          "expr": "$.someRecord.subInt"\n        }\n      ]\n    },\n    "binaryAsString": false\n  },\n  ...\n}\n')),(0,i.yg)("h5",{id:"avro-bytes-decoder"},"Avro Bytes Decoder"),(0,i.yg)("p",null,"If ",(0,i.yg)("inlineCode",{parentName:"p"},"type")," is not included, the avroBytesDecoder defaults to ",(0,i.yg)("inlineCode",{parentName:"p"},"schema_repo"),"."),(0,i.yg)("h6",{id:"inline-schema-based-avro-bytes-decoder"},"Inline Schema Based Avro Bytes Decoder"),(0,i.yg)("admonition",{type:"info"},(0,i.yg)("p",{parentName:"admonition"},' The "schema_inline" decoder reads Avro records using a fixed schema and does not support schema migration. If you\nmay need to migrate schemas in the future, consider one of the other decoders, all of which use a message header that\nallows the parser to identify the proper Avro schema for reading records.')),(0,i.yg)("p",null,"This decoder can be used if all the input events can be read using the same schema. In this case, specify the schema in the input task JSON itself, as described below."),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre"},'...\n"avroBytesDecoder": {\n  "type": "schema_inline",\n  "schema": {\n    //your schema goes here, for example\n    "namespace": "org.apache.druid.data",\n    "name": "User",\n    "type": "record",\n    "fields": [\n      { "name": "FullName", "type": "string" },\n      { "name": "Country", "type": "string" }\n    ]\n  }\n}\n...\n')),(0,i.yg)("h6",{id:"multiple-inline-schemas-based-avro-bytes-decoder"},"Multiple Inline Schemas Based Avro Bytes Decoder"),(0,i.yg)("p",null,"Use this decoder if different input events can have different read schemas. In this case, specify the schema in the input task JSON itself, as described below."),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre"},'...\n"avroBytesDecoder": {\n  "type": "multiple_schemas_inline",\n  "schemas": {\n    //your id -> schema map goes here, for example\n    "1": {\n      "namespace": "org.apache.druid.data",\n      "name": "User",\n      "type": "record",\n      "fields": [\n        { "name": "FullName", "type": "string" },\n        { "name": "Country", "type": "string" }\n      ]\n    },\n    "2": {\n      "namespace": "org.apache.druid.otherdata",\n      "name": "UserIdentity",\n      "type": "record",\n      "fields": [\n        { "name": "Name", "type": "string" },\n        { "name": "Location", "type": "string" }\n      ]\n    },\n    ...\n    ...\n  }\n}\n...\n')),(0,i.yg)("p",null,"Note that it is essentially a map of integer schema ID to avro schema object. This parser assumes that record has following format.\nfirst 1 byte is version and must always be 1.\nnext 4 bytes are integer schema ID serialized using big-endian byte order.\nremaining bytes contain serialized avro message."),(0,i.yg)("h5",{id:"schemarepo-based-avro-bytes-decoder"},"SchemaRepo Based Avro Bytes Decoder"),(0,i.yg)("p",null,"This Avro bytes decoder first extracts ",(0,i.yg)("inlineCode",{parentName:"p"},"subject")," and ",(0,i.yg)("inlineCode",{parentName:"p"},"id")," from the input message bytes, and then uses them to look up the Avro schema used to decode the Avro record from bytes. For details, see the ",(0,i.yg)("a",{parentName:"p",href:"https://github.com/schema-repo/schema-repo"},"schema repo"),". You need an HTTP service like schema repo to hold the Avro schema. For information on registering a schema on the message producer side, see ",(0,i.yg)("inlineCode",{parentName:"p"},"org.apache.druid.data.input.AvroStreamInputRowParserTest#testParse()"),"."),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"type"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"Set value to ",(0,i.yg)("inlineCode",{parentName:"td"},"schema_repo"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"no")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"subjectAndIdConverter"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies how to extract the subject and id from message bytes."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"schemaRepository"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies how to look up the Avro schema from subject and id."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")))),(0,i.yg)("h6",{id:"avro-1124-subject-and-id-converter"},"Avro-1124 Subject And Id Converter"),(0,i.yg)("p",null,"This section describes the format of the ",(0,i.yg)("inlineCode",{parentName:"p"},"subjectAndIdConverter")," object for the ",(0,i.yg)("inlineCode",{parentName:"p"},"schema_repo")," Avro bytes decoder."),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"type"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"Set value to ",(0,i.yg)("inlineCode",{parentName:"td"},"avro_1124"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"no")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"topic"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the topic of your Kafka stream."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")))),(0,i.yg)("h6",{id:"avro-1124-schema-repository"},"Avro-1124 Schema Repository"),(0,i.yg)("p",null,"This section describes the format of the ",(0,i.yg)("inlineCode",{parentName:"p"},"schemaRepository")," object for the ",(0,i.yg)("inlineCode",{parentName:"p"},"schema_repo")," Avro bytes decoder."),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"type"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"Set value to ",(0,i.yg)("inlineCode",{parentName:"td"},"avro_1124_rest_client"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"no")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"url"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the endpoint URL of your Avro-1124 schema repository."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")))),(0,i.yg)("h6",{id:"confluent-schema-registry-based-avro-bytes-decoder"},"Confluent Schema Registry-based Avro Bytes Decoder"),(0,i.yg)("p",null,"This Avro bytes decoder first extracts a unique ",(0,i.yg)("inlineCode",{parentName:"p"},"id")," from input message bytes, and then uses it to look up the schema in the Schema Registry used to decode the Avro record from bytes.\nFor details, see the Schema Registry ",(0,i.yg)("a",{parentName:"p",href:"http://docs.confluent.io/current/schema-registry/docs/"},"documentation")," and ",(0,i.yg)("a",{parentName:"p",href:"https://github.com/confluentinc/schema-registry"},"repository"),"."),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"type"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"Set value to ",(0,i.yg)("inlineCode",{parentName:"td"},"schema_registry"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"no")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"url"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the URL endpoint of the Schema Registry."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"capacity"),(0,i.yg)("td",{parentName:"tr",align:null},"Integer"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the max size of the cache (default = Integer.MAX_VALUE)."),(0,i.yg)("td",{parentName:"tr",align:null},"no")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"urls"),(0,i.yg)("td",{parentName:"tr",align:null},"Array<String",">"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the URL endpoints of the multiple Schema Registry instances."),(0,i.yg)("td",{parentName:"tr",align:null},"yes (if ",(0,i.yg)("inlineCode",{parentName:"td"},"url")," is not provided)")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"config"),(0,i.yg)("td",{parentName:"tr",align:null},"Json"),(0,i.yg)("td",{parentName:"tr",align:null},"To send additional configurations, configured for Schema Registry. This can be supplied via a ",(0,i.yg)("a",{parentName:"td",href:"/docs/latest/operations/dynamic-config-provider"},"DynamicConfigProvider")),(0,i.yg)("td",{parentName:"tr",align:null},"no")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"headers"),(0,i.yg)("td",{parentName:"tr",align:null},"Json"),(0,i.yg)("td",{parentName:"tr",align:null},"To send headers to the Schema Registry. This can be supplied via a ",(0,i.yg)("a",{parentName:"td",href:"/docs/latest/operations/dynamic-config-provider"},"DynamicConfigProvider")),(0,i.yg)("td",{parentName:"tr",align:null},"no")))),(0,i.yg)("p",null,"For a single schema registry instance, use Field ",(0,i.yg)("inlineCode",{parentName:"p"},"url")," or ",(0,i.yg)("inlineCode",{parentName:"p"},"urls")," for multi instances."),(0,i.yg)("p",null,"Single Instance:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'...\n"avroBytesDecoder" : {\n   "type" : "schema_registry",\n   "url" : <schema-registry-url>\n}\n...\n')),(0,i.yg)("p",null,"Multiple Instances:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'...\n"avroBytesDecoder" : {\n   "type" : "schema_registry",\n   "urls" : [<schema-registry-url-1>, <schema-registry-url-2>, ...],\n   "config" : {\n        "basic.auth.credentials.source": "USER_INFO",\n        "basic.auth.user.info": "fred:letmein",\n        "schema.registry.ssl.truststore.location": "/some/secrets/kafka.client.truststore.jks",\n        "schema.registry.ssl.truststore.password": "<password>",\n        "schema.registry.ssl.keystore.location": "/some/secrets/kafka.client.keystore.jks",\n        "schema.registry.ssl.keystore.password": "<password>",\n        "schema.registry.ssl.key.password": "<password>",\n        "schema.registry.ssl.key.password",\n       ... \n   },\n   "headers": {\n       "traceID" : "b29c5de2-0db4-490b-b421",\n       "timeStamp" : "1577191871865",\n       "druid.dynamic.config.provider":{\n            "type":"mapString", \n            "config":{\n                 "registry.header.prop.1":"value.1", \n                 "registry.header.prop.2":"value.2"\n                 }\n            }\n       ...\n    }\n}\n...\n')),(0,i.yg)("h6",{id:"parse-exceptions"},"Parse exceptions"),(0,i.yg)("p",null,"The following errors when reading records will be considered parse exceptions, which can be limited and logged with ingestion task configurations such as ",(0,i.yg)("inlineCode",{parentName:"p"},"maxParseExceptions")," and ",(0,i.yg)("inlineCode",{parentName:"p"},"maxSavedParseExceptions"),":"),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},"Failure to retrieve a schema due to misconfiguration or corrupt records (invalid schema IDs)"),(0,i.yg)("li",{parentName:"ul"},"Failure to decode an Avro message")),(0,i.yg)("h3",{id:"avro-ocf"},"Avro OCF"),(0,i.yg)("p",null,"To load the Avro OCF input format, load the Druid Avro extension (",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/development/extensions-core/avro"},(0,i.yg)("inlineCode",{parentName:"a"},"druid-avro-extensions")),")."),(0,i.yg)("p",null,"See the ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/development/extensions-core/avro#avro-types"},"Avro Types")," section for how Avro types are handled in Druid"),(0,i.yg)("p",null,"Configure the Avro OCF ",(0,i.yg)("inlineCode",{parentName:"p"},"inputFormat")," to load Avro OCF data as follows:"),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"type"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"Set value to ",(0,i.yg)("inlineCode",{parentName:"td"},"avro_ocf"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"flattenSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Define a ",(0,i.yg)("a",{parentName:"td",href:"#flattenspec"},(0,i.yg)("inlineCode",{parentName:"a"},"flattenSpec"))," to extract nested values from Avro records. Only 'path' expressions are supported ('jq' and 'tree' are unavailable)."),(0,i.yg)("td",{parentName:"tr",align:null},"no (default will auto-discover 'root' level properties)")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"schema"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Define a reader schema to be used when parsing Avro records. This is useful when parsing multiple versions of Avro OCF file data."),(0,i.yg)("td",{parentName:"tr",align:null},"no (default will decode using the writer schema contained in the OCF file)")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"binaryAsString"),(0,i.yg)("td",{parentName:"tr",align:null},"Boolean"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies if the bytes parquet column which is not logically marked as a string or enum type should be treated as a UTF-8 encoded string."),(0,i.yg)("td",{parentName:"tr",align:null},"no (default = false)")))),(0,i.yg)("p",null,"For example:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'"ioConfig": {\n  "inputFormat": {\n    "type": "avro_ocf",\n    "flattenSpec": {\n      "useFieldDiscovery": true,\n      "fields": [\n        {\n          "type": "path",\n          "name": "someRecord_subInt",\n          "expr": "$.someRecord.subInt"\n        }\n      ]\n    },\n    "schema": {\n      "namespace": "org.apache.druid.data.input",\n      "name": "SomeDatum",\n      "type": "record",\n      "fields" : [\n        { "name": "timestamp", "type": "long" },\n        { "name": "eventType", "type": "string" },\n        { "name": "id", "type": "long" },\n        { "name": "someRecord", "type": {\n          "type": "record", "name": "MySubRecord", "fields": [\n            { "name": "subInt", "type": "int"},\n            { "name": "subLong", "type": "long"}\n          ]\n        }}]\n    },\n    "binaryAsString": false\n  },\n  ...\n}\n')),(0,i.yg)("h3",{id:"protobuf"},"Protobuf"),(0,i.yg)("admonition",{type:"info"},(0,i.yg)("p",{parentName:"admonition"}," You need to include the ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/development/extensions-core/protobuf"},(0,i.yg)("inlineCode",{parentName:"a"},"druid-protobuf-extensions"))," as an extension to use the Protobuf input format.")),(0,i.yg)("p",null,"Configure the Protobuf ",(0,i.yg)("inlineCode",{parentName:"p"},"inputFormat")," to load Protobuf data as follows:"),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"type")),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"Set value to ",(0,i.yg)("inlineCode",{parentName:"td"},"protobuf"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"flattenSpec")),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Define a ",(0,i.yg)("a",{parentName:"td",href:"#flattenspec"},(0,i.yg)("inlineCode",{parentName:"a"},"flattenSpec"))," to extract nested values from a Protobuf record. Note that only 'path' expression are supported ('jq' and 'tree' is unavailable)."),(0,i.yg)("td",{parentName:"tr",align:null},"no (default will auto-discover 'root' level properties)")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"protoBytesDecoder")),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies how to decode bytes to Protobuf record."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")))),(0,i.yg)("p",null,"For example:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'"ioConfig": {\n  "inputFormat": {\n    "type": "protobuf",\n    "protoBytesDecoder": {\n      "type": "file",\n      "descriptor": "file:///tmp/metrics.desc",\n      "protoMessageType": "Metrics"\n    }\n    "flattenSpec": {\n      "useFieldDiscovery": true,\n      "fields": [\n        {\n          "type": "path",\n          "name": "someRecord_subInt",\n          "expr": "$.someRecord.subInt"\n        }\n      ]\n    }\n  },\n  ...\n}\n')),(0,i.yg)("h3",{id:"kafka"},"Kafka"),(0,i.yg)("p",null,"The ",(0,i.yg)("inlineCode",{parentName:"p"},"kafka")," input format lets you parse the Kafka metadata fields in addition to the Kafka payload value contents.\nIt should only be used when ingesting from Apache Kafka."),(0,i.yg)("p",null,"The ",(0,i.yg)("inlineCode",{parentName:"p"},"kafka")," input format wraps around the payload parsing input format and augments the data it outputs with the Kafka event timestamp, topic name, event headers, and the key field that itself can be parsed using any available input format."),(0,i.yg)("p",null,"If there are conflicts between column names in the payload and those created from the metadata, the payload takes precedence.\nThis ensures that upgrading a Kafka ingestion to use the Kafka input format (by taking its existing input format and setting it as the ",(0,i.yg)("inlineCode",{parentName:"p"},"valueFormat"),") can be done without losing any of the payload data.  "),(0,i.yg)("p",null,"Configure the Kafka ",(0,i.yg)("inlineCode",{parentName:"p"},"inputFormat")," as follows:"),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"),(0,i.yg)("th",{parentName:"tr",align:null},"Default"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"type")),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"Set value to ",(0,i.yg)("inlineCode",{parentName:"td"},"kafka"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"yes"),(0,i.yg)("td",{parentName:"tr",align:null})),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"valueFormat")),(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("a",{parentName:"td",href:"#input-format"},"InputFormat")),(0,i.yg)("td",{parentName:"tr",align:null},"The ",(0,i.yg)("a",{parentName:"td",href:"#input-format"},"input format")," to parse the Kafka value payload."),(0,i.yg)("td",{parentName:"tr",align:null},"yes"),(0,i.yg)("td",{parentName:"tr",align:null})),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"timestampColumnName")),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"The name of the column for the Kafka timestamp."),(0,i.yg)("td",{parentName:"tr",align:null},"no"),(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"kafka.timestamp"))),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"topicColumnName")),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"The name of the column for the Kafka topic. This field is useful when ingesting data from multiple topics into same datasource."),(0,i.yg)("td",{parentName:"tr",align:null},"no"),(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"kafka.topic"))),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"headerColumnPrefix")),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"The custom prefix for all the header columns."),(0,i.yg)("td",{parentName:"tr",align:null},"no"),(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"kafka.header"))),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"headerFormat")),(0,i.yg)("td",{parentName:"tr",align:null},"Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies how to parse the Kafka headers. Supports String types. Because Kafka header values are bytes, the parser decodes them as UTF-8 encoded strings. To change this behavior, implement your own parser based on the encoding style. Change the ",(0,i.yg)("inlineCode",{parentName:"td"},"encoding")," type in ",(0,i.yg)("inlineCode",{parentName:"td"},"KafkaStringHeaderFormat")," to match your custom implementation. See ",(0,i.yg)("a",{parentName:"td",href:"#header-format"},"Header format")," for supported encoding formats."),(0,i.yg)("td",{parentName:"tr",align:null},"no"),(0,i.yg)("td",{parentName:"tr",align:null})),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"keyFormat")),(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("a",{parentName:"td",href:"#input-format"},"InputFormat")),(0,i.yg)("td",{parentName:"tr",align:null},"The ",(0,i.yg)("a",{parentName:"td",href:"#input-format"},"input format")," to parse the Kafka key. It only processes the first entry of the ",(0,i.yg)("inlineCode",{parentName:"td"},"inputFormat")," field. If your key values are simple strings, you can use the ",(0,i.yg)("inlineCode",{parentName:"td"},"tsv")," format to parse them. Note that for ",(0,i.yg)("inlineCode",{parentName:"td"},"tsv"),",",(0,i.yg)("inlineCode",{parentName:"td"},"csv"),", and ",(0,i.yg)("inlineCode",{parentName:"td"},"regex")," formats, you need to provide a ",(0,i.yg)("inlineCode",{parentName:"td"},"columns")," array to make a valid input format. Only the first one is used, and its name will be ignored in favor of ",(0,i.yg)("inlineCode",{parentName:"td"},"keyColumnName"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"no"),(0,i.yg)("td",{parentName:"tr",align:null})),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"keyColumnName")),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"The name of the column for the Kafka key."),(0,i.yg)("td",{parentName:"tr",align:null},"no"),(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"kafka.key"))))),(0,i.yg)("h4",{id:"header-format"},"Header format"),(0,i.yg)("p",null,(0,i.yg)("inlineCode",{parentName:"p"},"headerFormat")," supports the following encoding formats:"),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"ISO-8859-1"),": ISO Latin Alphabet No. 1, that is, ISO-LATIN-1."),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"US-ASCII"),": Seven-bit ASCII. Also known as ISO646-US. The Basic Latin block of the Unicode character set."),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"UTF-8"),": Eight-bit UCS Transformation Format."),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"UTF-16"),": Sixteen-bit UCS Transformation Format, byte order identified by an optional byte-order mark."),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"UTF-16BE"),": Sixteen-bit UCS Transformation Format, big-endian byte order."),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"UTF-16LE"),": Sixteen-bit UCS Transformation Format, little-endian byte order."),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"headerColumnPrefix"),": Supply a prefix to the Kafka headers to avoid any conflicts with columns from the payload. The default is ",(0,i.yg)("inlineCode",{parentName:"li"},"kafka.header."),".")),(0,i.yg)("h4",{id:"example"},"Example"),(0,i.yg)("p",null,"Using ",(0,i.yg)("inlineCode",{parentName:"p"},'{ "type": "json" }')," as the input format would only parse the payload value.\nTo parse the Kafka metadata in addition to the payload, use the ",(0,i.yg)("inlineCode",{parentName:"p"},"kafka")," input format."),(0,i.yg)("p",null,"For example, consider the following structure for a Kafka message that represents an edit in a development environment:"),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("strong",{parentName:"li"},"Kafka timestamp"),": ",(0,i.yg)("inlineCode",{parentName:"li"},"1680795276351")),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("strong",{parentName:"li"},"Kafka topic"),": ",(0,i.yg)("inlineCode",{parentName:"li"},"wiki-edits")),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("strong",{parentName:"li"},"Kafka headers"),":",(0,i.yg)("ul",{parentName:"li"},(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"env=development")),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"zone=z1")))),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("strong",{parentName:"li"},"Kafka key"),": ",(0,i.yg)("inlineCode",{parentName:"li"},"wiki-edit")),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("strong",{parentName:"li"},"Kafka payload value"),": ",(0,i.yg)("inlineCode",{parentName:"li"},'{"channel":"#sv.wikipedia","timestamp":"2016-06-27T00:00:11.080Z","page":"Salo Toraut","delta":31,"namespace":"Main"}'))),(0,i.yg)("p",null,"You would configure it as follows:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'"ioConfig": {\n  "inputFormat": {\n    "type": "kafka",\n    "valueFormat": {\n      "type": "json"\n    },\n    "timestampColumnName": "kafka.timestamp",\n    "topicColumnName": "kafka.topic",\n    "headerFormat": {\n      "type": "string",\n      "encoding": "UTF-8"\n    },\n    "headerColumnPrefix": "kafka.header.",\n    "keyFormat": {\n      "type": "tsv",\n      "findColumnsFromHeader": false,\n      "columns": ["x"]\n    },\n    "keyColumnName": "kafka.key",\n  }\n}\n')),(0,i.yg)("p",null,"You would parse the example message as follows:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'{\n  "channel": "#sv.wikipedia",\n  "timestamp": "2016-06-27T00:00:11.080Z",\n  "page": "Salo Toraut",\n  "delta": 31,\n  "namespace": "Main",\n  "kafka.timestamp": 1680795276351,\n  "kafka.topic": "wiki-edits",\n  "kafka.header.env": "development",\n  "kafka.header.zone": "z1",\n  "kafka.key": "wiki-edit"\n}\n')),(0,i.yg)("p",null,"If you want to use ",(0,i.yg)("inlineCode",{parentName:"p"},"kafka.timestamp")," as Druid's primary timestamp (",(0,i.yg)("inlineCode",{parentName:"p"},"__time"),"), specify it as the value for ",(0,i.yg)("inlineCode",{parentName:"p"},"column")," in the ",(0,i.yg)("inlineCode",{parentName:"p"},"timestampSpec"),":"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'"timestampSpec": {\n  "column": "kafka.timestamp",\n  "format": "millis"\n}\n')),(0,i.yg)("p",null,"Similarly, if you want to use a timestamp extracted from the Kafka header:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'"timestampSpec": {\n  "column": "kafka.header.myTimestampHeader",\n  "format": "millis"\n}\n')),(0,i.yg)("p",null,"Finally, add these Kafka metadata columns to the ",(0,i.yg)("inlineCode",{parentName:"p"},"dimensionsSpec")," or  set your ",(0,i.yg)("inlineCode",{parentName:"p"},"dimensionsSpec")," to auto-detect columns."),(0,i.yg)("p",null,"The following supervisor spec demonstrates how to ingest the Kafka header, key, timestamp, and topic into Druid dimensions:"),(0,i.yg)("details",null,(0,i.yg)("summary",null,"Click to view the example"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'{\n  "type": "kafka",\n  "spec": {\n    "ioConfig": {\n      "type": "kafka",\n      "consumerProperties": {\n        "bootstrap.servers": "localhost:9092"\n      },\n      "topic": "wiki-edits",\n      "inputFormat": {\n        "type": "kafka",\n        "valueFormat": {\n          "type": "json"\n        },\n        "headerFormat": {\n          "type": "string"\n        },\n        "keyFormat": {\n          "type": "tsv",\n          "findColumnsFromHeader": false,\n          "columns": ["x"]\n        }\n      },\n      "useEarliestOffset": true\n    },\n    "dataSchema": {\n      "dataSource": "wikiticker",\n      "timestampSpec": {\n        "column": "timestamp",\n        "format": "posix"\n      },\n      "dimensionsSpec":  "dimensionsSpec": {\n        "useSchemaDiscovery": true,\n        "includeAllDimensions": true\n      },\n      "granularitySpec": {\n        "queryGranularity": "none",\n        "rollup": false,\n        "segmentGranularity": "day"\n      }\n    },\n    "tuningConfig": {\n      "type": "kafka"\n    }\n  }\n}\n'))),(0,i.yg)("p",null,"After Druid ingests the data, you can query the Kafka metadata columns as follows:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-sql"},'SELECT\n  "kafka.header.env",\n  "kafka.key",\n  "kafka.timestamp",\n  "kafka.topic"\nFROM "wikiticker"\n')),(0,i.yg)("p",null,"This query returns:"),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"th"},"kafka.header.env")),(0,i.yg)("th",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"th"},"kafka.key")),(0,i.yg)("th",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"th"},"kafka.timestamp")),(0,i.yg)("th",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"th"},"kafka.topic")))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"development")),(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"wiki-edit")),(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"1680795276351")),(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"wiki-edits"))))),(0,i.yg)("h2",{id:"flattenspec"},"FlattenSpec"),(0,i.yg)("p",null,"You can use the ",(0,i.yg)("inlineCode",{parentName:"p"},"flattenSpec")," object to flatten nested data, as an alternative to the Druid ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/querying/nested-columns"},"nested columns")," feature, and for nested input formats unsupported by the feature. It is an object within the ",(0,i.yg)("inlineCode",{parentName:"p"},"inputFormat")," object."),(0,i.yg)("p",null,"See ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/querying/nested-columns"},"Nested columns")," for information on ingesting and storing nested data in an Apache Druid column as a ",(0,i.yg)("inlineCode",{parentName:"p"},"COMPLEX<json>")," data type."),(0,i.yg)("p",null,"Configure your ",(0,i.yg)("inlineCode",{parentName:"p"},"flattenSpec")," as follows:"),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Default"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"useFieldDiscovery"),(0,i.yg)("td",{parentName:"tr",align:null},"If true, interpret all root-level fields as available fields for usage by ",(0,i.yg)("a",{parentName:"td",href:"/docs/latest/ingestion/ingestion-spec#timestampspec"},(0,i.yg)("inlineCode",{parentName:"a"},"timestampSpec")),", ",(0,i.yg)("a",{parentName:"td",href:"/docs/latest/ingestion/ingestion-spec#transformspec"},(0,i.yg)("inlineCode",{parentName:"a"},"transformSpec")),", ",(0,i.yg)("a",{parentName:"td",href:"/docs/latest/ingestion/ingestion-spec#dimensionsspec"},(0,i.yg)("inlineCode",{parentName:"a"},"dimensionsSpec")),", and ",(0,i.yg)("a",{parentName:"td",href:"/docs/latest/ingestion/ingestion-spec#metricsspec"},(0,i.yg)("inlineCode",{parentName:"a"},"metricsSpec")),".",(0,i.yg)("br",null),(0,i.yg)("br",null),"If false, only explicitly specified fields (see ",(0,i.yg)("inlineCode",{parentName:"td"},"fields"),") will be available for use."),(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"true"))),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"fields"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the fields of interest and how they are accessed. See ",(0,i.yg)("a",{parentName:"td",href:"#field-flattening-specifications"},"Field flattening specifications")," for more detail."),(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"[]"))))),(0,i.yg)("p",null,"For example:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'"flattenSpec": {\n  "useFieldDiscovery": true,\n  "fields": [\n    { "name": "baz", "type": "root" },\n    { "name": "foo_bar", "type": "path", "expr": "$.foo.bar" },\n    { "name": "foo_other_bar", "type": "tree", "nodes": ["foo", "other", "bar"] },\n    { "name": "first_food", "type": "jq", "expr": ".thing.food[1]" }\n  ]\n}\n')),(0,i.yg)("p",null,"After Druid reads the input data records, it applies the flattenSpec before applying any other specs such as ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/ingestion/ingestion-spec#timestampspec"},(0,i.yg)("inlineCode",{parentName:"a"},"timestampSpec")),", ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/ingestion/ingestion-spec#transformspec"},(0,i.yg)("inlineCode",{parentName:"a"},"transformSpec")),", ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/ingestion/ingestion-spec#dimensionsspec"},(0,i.yg)("inlineCode",{parentName:"a"},"dimensionsSpec")),", or ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/ingestion/ingestion-spec#metricsspec"},(0,i.yg)("inlineCode",{parentName:"a"},"metricsSpec")),".  This makes it possible to extract timestamps from flattened data, for example, and to refer to flattened data in transformations, in your dimension list, and when generating metrics."),(0,i.yg)("p",null,"Flattening is only supported for ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/ingestion/data-formats"},"data formats")," that support nesting, including ",(0,i.yg)("inlineCode",{parentName:"p"},"avro"),", ",(0,i.yg)("inlineCode",{parentName:"p"},"json"),", ",(0,i.yg)("inlineCode",{parentName:"p"},"orc"),", and ",(0,i.yg)("inlineCode",{parentName:"p"},"parquet"),"."),(0,i.yg)("h3",{id:"field-flattening-specifications"},"Field flattening specifications"),(0,i.yg)("p",null,"Each entry in the ",(0,i.yg)("inlineCode",{parentName:"p"},"fields")," list can have the following components:"),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Default"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"type"),(0,i.yg)("td",{parentName:"tr",align:null},"Options are as follows:",(0,i.yg)("br",null),(0,i.yg)("br",null),(0,i.yg)("ul",null,(0,i.yg)("li",null,(0,i.yg)("inlineCode",{parentName:"td"},"root"),", referring to a field at the root level of the record. Only really useful if ",(0,i.yg)("inlineCode",{parentName:"td"},"useFieldDiscovery")," is false."),(0,i.yg)("li",null,(0,i.yg)("inlineCode",{parentName:"td"},"path"),", referring to a field using ",(0,i.yg)("a",{parentName:"td",href:"https://github.com/jayway/JsonPath"},"JsonPath")," notation. Supported by most data formats that offer nesting, including ",(0,i.yg)("inlineCode",{parentName:"td"},"avro"),", ",(0,i.yg)("inlineCode",{parentName:"td"},"json"),", ",(0,i.yg)("inlineCode",{parentName:"td"},"orc"),", and ",(0,i.yg)("inlineCode",{parentName:"td"},"parquet"),"."),(0,i.yg)("li",null,(0,i.yg)("inlineCode",{parentName:"td"},"jq"),", referring to a field using ",(0,i.yg)("a",{parentName:"td",href:"https://github.com/eiiches/jackson-jq"},"jackson-jq")," notation. Only supported for the ",(0,i.yg)("inlineCode",{parentName:"td"},"json")," format."),(0,i.yg)("li",null,(0,i.yg)("inlineCode",{parentName:"td"},"tree"),", referring to a nested field from the root level of the record. Useful and more efficient than ",(0,i.yg)("inlineCode",{parentName:"td"},"path")," or ",(0,i.yg)("inlineCode",{parentName:"td"},"jq")," if a simple hierarchical fetch is required. Only supported for the ",(0,i.yg)("inlineCode",{parentName:"td"},"json")," format."))),(0,i.yg)("td",{parentName:"tr",align:null},"none (required)")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"name"),(0,i.yg)("td",{parentName:"tr",align:null},"Name of the field after flattening. This name can be referred to by the ",(0,i.yg)("a",{parentName:"td",href:"/docs/latest/ingestion/ingestion-spec#timestampspec"},(0,i.yg)("inlineCode",{parentName:"a"},"timestampSpec")),", ",(0,i.yg)("a",{parentName:"td",href:"/docs/latest/ingestion/ingestion-spec#transformspec"},(0,i.yg)("inlineCode",{parentName:"a"},"transformSpec")),", ",(0,i.yg)("a",{parentName:"td",href:"/docs/latest/ingestion/ingestion-spec#dimensionsspec"},(0,i.yg)("inlineCode",{parentName:"a"},"dimensionsSpec")),", and ",(0,i.yg)("a",{parentName:"td",href:"/docs/latest/ingestion/ingestion-spec#metricsspec"},(0,i.yg)("inlineCode",{parentName:"a"},"metricsSpec")),"."),(0,i.yg)("td",{parentName:"tr",align:null},"none (required)")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"expr"),(0,i.yg)("td",{parentName:"tr",align:null},"Expression for accessing the field while flattening. For type ",(0,i.yg)("inlineCode",{parentName:"td"},"path"),", this should be ",(0,i.yg)("a",{parentName:"td",href:"https://github.com/jayway/JsonPath"},"JsonPath"),". For type ",(0,i.yg)("inlineCode",{parentName:"td"},"jq"),", this should be ",(0,i.yg)("a",{parentName:"td",href:"https://github.com/eiiches/jackson-jq"},"jackson-jq")," notation. For other types, this parameter is ignored."),(0,i.yg)("td",{parentName:"tr",align:null},"none (required for types ",(0,i.yg)("inlineCode",{parentName:"td"},"path")," and ",(0,i.yg)("inlineCode",{parentName:"td"},"jq"),")")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"nodes"),(0,i.yg)("td",{parentName:"tr",align:null},"For ",(0,i.yg)("inlineCode",{parentName:"td"},"tree")," only. Multiple-expression field for accessing the field while flattening, representing the hierarchy of field names to read. For other types, this parameter must not be provided."),(0,i.yg)("td",{parentName:"tr",align:null},"none (required for type ",(0,i.yg)("inlineCode",{parentName:"td"},"tree"),")")))),(0,i.yg)("h3",{id:"notes-on-flattening"},"Notes on flattening"),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("p",{parentName:"li"},"For convenience, when defining a root-level field, it is possible to define only the field name, as a string, instead of a JSON object. For example, ",(0,i.yg)("inlineCode",{parentName:"p"},'{"name": "baz", "type": "root"}')," is equivalent to ",(0,i.yg)("inlineCode",{parentName:"p"},'"baz"'),".")),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("p",{parentName:"li"},"Enabling ",(0,i.yg)("inlineCode",{parentName:"p"},"useFieldDiscovery"),' will only automatically detect "simple" fields at the root level that correspond to data types that Druid supports. This includes strings, numbers, and lists of strings or numbers. Other types will not be automatically detected, and must be specified explicitly in the ',(0,i.yg)("inlineCode",{parentName:"p"},"fields")," list.")),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("p",{parentName:"li"},"Duplicate field ",(0,i.yg)("inlineCode",{parentName:"p"},"name"),"s are not allowed. An exception will be thrown.")),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("p",{parentName:"li"},"If ",(0,i.yg)("inlineCode",{parentName:"p"},"useFieldDiscovery")," is enabled, any discovered field with the same name as one already defined in the ",(0,i.yg)("inlineCode",{parentName:"p"},"fields")," list will be skipped, rather than added twice.")),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("p",{parentName:"li"},(0,i.yg)("a",{parentName:"p",href:"https://jsonpath.com/"},"JSONPath evaluator")," is useful for testing ",(0,i.yg)("inlineCode",{parentName:"p"},"path"),"-type expressions.")),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("p",{parentName:"li"},"jackson-jq supports a subset of the full ",(0,i.yg)("a",{parentName:"p",href:"https://stedolan.github.io/jq/"},"jq")," syntax.  Please refer to the ",(0,i.yg)("a",{parentName:"p",href:"https://github.com/eiiches/jackson-jq"},"jackson-jq documentation")," for details.")),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("p",{parentName:"li"},(0,i.yg)("a",{parentName:"p",href:"https://github.com/jayway/JsonPath"},"JsonPath")," supports a bunch of functions, but not all of these functions are supported by Druid now. Following matrix shows the current supported JsonPath functions and corresponding data formats. Please also note the output data type of these functions."),(0,i.yg)("table",{parentName:"li"},(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:"left"},"Function"),(0,i.yg)("th",{parentName:"tr",align:"left"},"Description"),(0,i.yg)("th",{parentName:"tr",align:"left"},"Output type"),(0,i.yg)("th",{parentName:"tr",align:"left"},"json"),(0,i.yg)("th",{parentName:"tr",align:"left"},"orc"),(0,i.yg)("th",{parentName:"tr",align:"left"},"avro"),(0,i.yg)("th",{parentName:"tr",align:"left"},"parquet"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:"left"},"min()"),(0,i.yg)("td",{parentName:"tr",align:"left"},"Provides the min value of an array of numbers"),(0,i.yg)("td",{parentName:"tr",align:"left"},"Double"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2713"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2713"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2713"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2713")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:"left"},"max()"),(0,i.yg)("td",{parentName:"tr",align:"left"},"Provides the max value of an array of numbers"),(0,i.yg)("td",{parentName:"tr",align:"left"},"Double"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2713"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2713"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2713"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2713")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:"left"},"avg()"),(0,i.yg)("td",{parentName:"tr",align:"left"},"Provides the average value of an array of numbers"),(0,i.yg)("td",{parentName:"tr",align:"left"},"Double"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2713"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2713"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2713"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2713")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:"left"},"stddev()"),(0,i.yg)("td",{parentName:"tr",align:"left"},"Provides the standard deviation value of an array of numbers"),(0,i.yg)("td",{parentName:"tr",align:"left"},"Double"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2713"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2713"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2713"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2713")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:"left"},"length()"),(0,i.yg)("td",{parentName:"tr",align:"left"},"Provides the length of an array"),(0,i.yg)("td",{parentName:"tr",align:"left"},"Integer"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2713"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2713"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2713"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2713")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:"left"},"sum()"),(0,i.yg)("td",{parentName:"tr",align:"left"},"Provides the sum value of an array of numbers"),(0,i.yg)("td",{parentName:"tr",align:"left"},"Double"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2713"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2713"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2713"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2713")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:"left"},"concat(X)"),(0,i.yg)("td",{parentName:"tr",align:"left"},"Provides a concatenated version of the path output with a new item"),(0,i.yg)("td",{parentName:"tr",align:"left"},"like input"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2713"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2717"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2717"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2717")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:"left"},"append(X)"),(0,i.yg)("td",{parentName:"tr",align:"left"},"add an item to the json path output array"),(0,i.yg)("td",{parentName:"tr",align:"left"},"like input"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2713"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2717"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2717"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2717")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:"left"},"keys()"),(0,i.yg)("td",{parentName:"tr",align:"left"},"Provides the property keys (An alternative for terminal tilde ~)"),(0,i.yg)("td",{parentName:"tr",align:"left"},"Set<E",">"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2717"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2717"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2717"),(0,i.yg)("td",{parentName:"tr",align:"left"},"\u2717")))))),(0,i.yg)("h2",{id:"parser"},"Parser"),(0,i.yg)("admonition",{type:"info"},(0,i.yg)("p",{parentName:"admonition"}," The Parser is deprecated for ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/ingestion/native-batch"},"native batch tasks"),", ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/ingestion/kafka-ingestion"},"Kafka indexing service"),",\nand ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/ingestion/kinesis-ingestion"},"Kinesis indexing service"),".\nConsider using the ",(0,i.yg)("a",{parentName:"p",href:"#input-format"},"input format")," instead for these types of ingestion.")),(0,i.yg)("p",null,"This section lists all default and core extension parsers.\nFor community extension parsers, please see our ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/configuration/extensions#community-extensions"},"community extensions list"),"."),(0,i.yg)("h3",{id:"string-parser"},"String Parser"),(0,i.yg)("p",null,(0,i.yg)("inlineCode",{parentName:"p"},"string")," typed parsers operate on text based inputs that can be split into individual records by newlines.\nEach line can be further parsed using ",(0,i.yg)("a",{parentName:"p",href:"#parsespec"},(0,i.yg)("inlineCode",{parentName:"a"},"parseSpec")),"."),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"type"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"Set value to ",(0,i.yg)("inlineCode",{parentName:"td"},"string")," for most cases. Otherwise use ",(0,i.yg)("inlineCode",{parentName:"td"},"hadoopyString")," for Hadoop indexing."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"parseSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the format, timestamp, and dimensions of the data."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")))),(0,i.yg)("h3",{id:"avro-hadoop-parser"},"Avro Hadoop Parser"),(0,i.yg)("admonition",{type:"info"},(0,i.yg)("p",{parentName:"admonition"}," You need to include the ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/development/extensions-core/avro"},(0,i.yg)("inlineCode",{parentName:"a"},"druid-avro-extensions"))," as an extension to use the Avro Hadoop Parser.")),(0,i.yg)("admonition",{type:"info"},(0,i.yg)("p",{parentName:"admonition"}," See the ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/development/extensions-core/avro#avro-types"},"Avro Types")," section for how Avro types are handled in Druid")),(0,i.yg)("p",null,"This parser is for ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/ingestion/hadoop"},"Hadoop batch ingestion"),".\nThe ",(0,i.yg)("inlineCode",{parentName:"p"},"inputFormat")," of ",(0,i.yg)("inlineCode",{parentName:"p"},"inputSpec")," in ",(0,i.yg)("inlineCode",{parentName:"p"},"ioConfig")," must be set to ",(0,i.yg)("inlineCode",{parentName:"p"},'"org.apache.druid.data.input.avro.AvroValueInputFormat"'),".\nYou may want to set Avro reader's schema in ",(0,i.yg)("inlineCode",{parentName:"p"},"jobProperties")," in ",(0,i.yg)("inlineCode",{parentName:"p"},"tuningConfig"),",\ne.g.: ",(0,i.yg)("inlineCode",{parentName:"p"},'"avro.schema.input.value.path": "/path/to/your/schema.avsc"')," or\n",(0,i.yg)("inlineCode",{parentName:"p"},'"avro.schema.input.value": "your_schema_JSON_object"'),".\nIf the Avro reader's schema is not set, the schema in Avro object container file will be used.\nSee ",(0,i.yg)("a",{parentName:"p",href:"http://avro.apache.org/docs/1.7.7/spec.html#Schema+Resolution"},"Avro specification")," for more information."),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"type"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"Set value to ",(0,i.yg)("inlineCode",{parentName:"td"},"avro_hadoop"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"parseSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},'Specifies the timestamp and dimensions of the data. Should be an "avro" parseSpec.'),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"fromPigAvroStorage"),(0,i.yg)("td",{parentName:"tr",align:null},"Boolean"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies whether the data file is stored using AvroStorage."),(0,i.yg)("td",{parentName:"tr",align:null},"no(default == false)")))),(0,i.yg)("p",null,"An Avro parseSpec can contain a ",(0,i.yg)("a",{parentName:"p",href:"#flattenspec"},(0,i.yg)("inlineCode",{parentName:"a"},"flattenSpec")),' using either the "root" or "path"\nfield types, which can be used to read nested Avro records. The "jq" and "tree" field type is not currently supported\nfor Avro.'),(0,i.yg)("p",null,"For example, using Avro Hadoop parser with custom reader's schema file:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'{\n  "type" : "index_hadoop",\n  "spec" : {\n    "dataSchema" : {\n      "dataSource" : "",\n      "parser" : {\n        "type" : "avro_hadoop",\n        "parseSpec" : {\n          "format": "avro",\n          "timestampSpec": <standard timestampSpec>,\n          "dimensionsSpec": <standard dimensionsSpec>,\n          "flattenSpec": <optional>\n        }\n      }\n    },\n    "ioConfig" : {\n      "type" : "hadoop",\n      "inputSpec" : {\n        "type" : "static",\n        "inputFormat": "org.apache.druid.data.input.avro.AvroValueInputFormat",\n        "paths" : ""\n      }\n    },\n    "tuningConfig" : {\n       "jobProperties" : {\n          "avro.schema.input.value.path" : "/path/to/my/schema.avsc"\n      }\n    }\n  }\n}\n')),(0,i.yg)("h3",{id:"orc-hadoop-parser"},"ORC Hadoop Parser"),(0,i.yg)("admonition",{type:"info"},(0,i.yg)("p",{parentName:"admonition"}," You need to include the ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/development/extensions-core/orc"},(0,i.yg)("inlineCode",{parentName:"a"},"druid-orc-extensions"))," as an extension to use the ORC Hadoop Parser.")),(0,i.yg)("admonition",{type:"info"},(0,i.yg)("p",{parentName:"admonition"}," If you are considering upgrading from earlier than 0.15.0 to 0.15.0 or a higher version,\nplease read ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/development/extensions-core/orc#migration-from-contrib-extension"},"Migration from 'contrib' extension")," carefully.")),(0,i.yg)("p",null,"This parser is for ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/ingestion/hadoop"},"Hadoop batch ingestion"),".\nThe ",(0,i.yg)("inlineCode",{parentName:"p"},"inputFormat")," of ",(0,i.yg)("inlineCode",{parentName:"p"},"inputSpec")," in ",(0,i.yg)("inlineCode",{parentName:"p"},"ioConfig")," must be set to ",(0,i.yg)("inlineCode",{parentName:"p"},'"org.apache.orc.mapreduce.OrcInputFormat"'),"."),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"type"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"Set value to ",(0,i.yg)("inlineCode",{parentName:"td"},"orc"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"parseSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the timestamp and dimensions of the data (",(0,i.yg)("inlineCode",{parentName:"td"},"timeAndDims")," and ",(0,i.yg)("inlineCode",{parentName:"td"},"orc")," format) and a ",(0,i.yg)("inlineCode",{parentName:"td"},"flattenSpec")," (",(0,i.yg)("inlineCode",{parentName:"td"},"orc")," format)."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")))),(0,i.yg)("p",null,"The parser supports two ",(0,i.yg)("inlineCode",{parentName:"p"},"parseSpec")," formats: ",(0,i.yg)("inlineCode",{parentName:"p"},"orc")," and ",(0,i.yg)("inlineCode",{parentName:"p"},"timeAndDims"),"."),(0,i.yg)("p",null,(0,i.yg)("inlineCode",{parentName:"p"},"orc")," supports auto field discovery and flattening, if specified with a ",(0,i.yg)("a",{parentName:"p",href:"#flattenspec"},(0,i.yg)("inlineCode",{parentName:"a"},"flattenSpec")),".\nIf no ",(0,i.yg)("inlineCode",{parentName:"p"},"flattenSpec")," is specified, ",(0,i.yg)("inlineCode",{parentName:"p"},"useFieldDiscovery")," will be enabled by default. Specifying a ",(0,i.yg)("inlineCode",{parentName:"p"},"dimensionSpec")," is\noptional if ",(0,i.yg)("inlineCode",{parentName:"p"},"useFieldDiscovery")," is enabled: if a ",(0,i.yg)("inlineCode",{parentName:"p"},"dimensionSpec")," is supplied, the list of ",(0,i.yg)("inlineCode",{parentName:"p"},"dimensions")," it defines will be\nthe set of ingested dimensions, if missing the discovered fields will make up the list."),(0,i.yg)("p",null,(0,i.yg)("inlineCode",{parentName:"p"},"timeAndDims")," parse spec must specify which fields will be extracted as dimensions through the ",(0,i.yg)("inlineCode",{parentName:"p"},"dimensionSpec"),"."),(0,i.yg)("p",null,(0,i.yg)("a",{parentName:"p",href:"https://orc.apache.org/docs/types.html"},"All column types")," are supported, with the exception of ",(0,i.yg)("inlineCode",{parentName:"p"},"union")," types. Columns of\n",(0,i.yg)("inlineCode",{parentName:"p"},"list")," type, if filled with primitives, may be used as a multi-value dimension, or specific elements can be extracted with\n",(0,i.yg)("inlineCode",{parentName:"p"},"flattenSpec")," expressions. Likewise, primitive fields may be extracted from ",(0,i.yg)("inlineCode",{parentName:"p"},"map")," and ",(0,i.yg)("inlineCode",{parentName:"p"},"struct")," types in the same manner.\nAuto field discovery will automatically create a string dimension for every (non-timestamp) primitive or ",(0,i.yg)("inlineCode",{parentName:"p"},"list")," of\nprimitives, as well as any flatten expressions defined in the ",(0,i.yg)("inlineCode",{parentName:"p"},"flattenSpec"),"."),(0,i.yg)("h4",{id:"hadoop-job-properties"},"Hadoop job properties"),(0,i.yg)("p",null,"Like most Hadoop jobs, the best outcomes will add ",(0,i.yg)("inlineCode",{parentName:"p"},'"mapreduce.job.user.classpath.first": "true"')," or\n",(0,i.yg)("inlineCode",{parentName:"p"},'"mapreduce.job.classloader": "true"')," to the ",(0,i.yg)("inlineCode",{parentName:"p"},"jobProperties")," section of ",(0,i.yg)("inlineCode",{parentName:"p"},"tuningConfig"),". Note that it is likely if using\n",(0,i.yg)("inlineCode",{parentName:"p"},'"mapreduce.job.classloader": "true"')," that you will need to set ",(0,i.yg)("inlineCode",{parentName:"p"},"mapreduce.job.classloader.system.classes")," to include\n",(0,i.yg)("inlineCode",{parentName:"p"},"-org.apache.hadoop.hive.")," to instruct Hadoop to load ",(0,i.yg)("inlineCode",{parentName:"p"},"org.apache.hadoop.hive")," classes from the application jars instead\nof system jars, e.g."),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'...\n    "mapreduce.job.classloader": "true",\n    "mapreduce.job.classloader.system.classes" : "java., javax.accessibility., javax.activation., javax.activity., javax.annotation., javax.annotation.processing., javax.crypto., javax.imageio., javax.jws., javax.lang.model., -javax.management.j2ee., javax.management., javax.naming., javax.net., javax.print., javax.rmi., javax.script., -javax.security.auth.message., javax.security.auth., javax.security.cert., javax.security.sasl., javax.sound., javax.sql., javax.swing., javax.tools., javax.transaction., -javax.xml.registry., -javax.xml.rpc., javax.xml., org.w3c.dom., org.xml.sax., org.apache.commons.logging., org.apache.log4j., -org.apache.hadoop.hbase., -org.apache.hadoop.hive., org.apache.hadoop., core-default.xml, hdfs-default.xml, mapred-default.xml, yarn-default.xml",\n...\n')),(0,i.yg)("p",null,"This is due to the ",(0,i.yg)("inlineCode",{parentName:"p"},"hive-storage-api")," dependency of the\n",(0,i.yg)("inlineCode",{parentName:"p"},"orc-mapreduce")," library, which provides some classes under the ",(0,i.yg)("inlineCode",{parentName:"p"},"org.apache.hadoop.hive")," package. If instead using the\nsetting ",(0,i.yg)("inlineCode",{parentName:"p"},'"mapreduce.job.user.classpath.first": "true"'),", then this will not be an issue."),(0,i.yg)("h4",{id:"examples"},"Examples"),(0,i.yg)("h5",{id:"orc-parser-orc-parsespec-auto-field-discovery-flatten-expressions"},(0,i.yg)("inlineCode",{parentName:"h5"},"orc")," parser, ",(0,i.yg)("inlineCode",{parentName:"h5"},"orc")," parseSpec, auto field discovery, flatten expressions"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'{\n  "type": "index_hadoop",\n  "spec": {\n    "ioConfig": {\n      "type": "hadoop",\n      "inputSpec": {\n        "type": "static",\n        "inputFormat": "org.apache.orc.mapreduce.OrcInputFormat",\n        "paths": "path/to/file.orc"\n      },\n      ...\n    },\n    "dataSchema": {\n      "dataSource": "example",\n      "parser": {\n        "type": "orc",\n        "parseSpec": {\n          "format": "orc",\n          "flattenSpec": {\n            "useFieldDiscovery": true,\n            "fields": [\n              {\n                "type": "path",\n                "name": "nestedDim",\n                "expr": "$.nestedData.dim1"\n              },\n              {\n                "type": "path",\n                "name": "listDimFirstItem",\n                "expr": "$.listDim[1]"\n              }\n            ]\n          },\n          "timestampSpec": {\n            "column": "timestamp",\n            "format": "millis"\n          }\n        }\n      },\n      ...\n    },\n    "tuningConfig": <hadoop-tuning-config>\n    }\n  }\n}\n')),(0,i.yg)("h5",{id:"orc-parser-orc-parsespec-field-discovery-with-no-flattenspec-or-dimensionspec"},(0,i.yg)("inlineCode",{parentName:"h5"},"orc")," parser, ",(0,i.yg)("inlineCode",{parentName:"h5"},"orc")," parseSpec, field discovery with no flattenSpec or dimensionSpec"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'{\n  "type": "index_hadoop",\n  "spec": {\n    "ioConfig": {\n      "type": "hadoop",\n      "inputSpec": {\n        "type": "static",\n        "inputFormat": "org.apache.orc.mapreduce.OrcInputFormat",\n        "paths": "path/to/file.orc"\n      },\n      ...\n    },\n    "dataSchema": {\n      "dataSource": "example",\n      "parser": {\n        "type": "orc",\n        "parseSpec": {\n          "format": "orc",\n          "timestampSpec": {\n            "column": "timestamp",\n            "format": "millis"\n          }\n        }\n      },\n      ...\n    },\n    "tuningConfig": <hadoop-tuning-config>\n    }\n  }\n}\n')),(0,i.yg)("h5",{id:"orc-parser-orc-parsespec-no-autodiscovery"},(0,i.yg)("inlineCode",{parentName:"h5"},"orc")," parser, ",(0,i.yg)("inlineCode",{parentName:"h5"},"orc")," parseSpec, no autodiscovery"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'{\n  "type": "index_hadoop",\n  "spec": {\n    "ioConfig": {\n      "type": "hadoop",\n      "inputSpec": {\n        "type": "static",\n        "inputFormat": "org.apache.orc.mapreduce.OrcInputFormat",\n        "paths": "path/to/file.orc"\n      },\n      ...\n    },\n    "dataSchema": {\n      "dataSource": "example",\n      "parser": {\n        "type": "orc",\n        "parseSpec": {\n          "format": "orc",\n          "flattenSpec": {\n            "useFieldDiscovery": false,\n            "fields": [\n              {\n                "type": "path",\n                "name": "nestedDim",\n                "expr": "$.nestedData.dim1"\n              },\n              {\n                "type": "path",\n                "name": "listDimFirstItem",\n                "expr": "$.listDim[1]"\n              }\n            ]\n          },\n          "timestampSpec": {\n            "column": "timestamp",\n            "format": "millis"\n          },\n          "dimensionsSpec": {\n            "dimensions": [\n              "dim1",\n              "dim3",\n              "nestedDim",\n              "listDimFirstItem"\n            ],\n            "dimensionExclusions": [],\n            "spatialDimensions": []\n          }\n        }\n      },\n      ...\n    },\n    "tuningConfig": <hadoop-tuning-config>\n    }\n  }\n}\n')),(0,i.yg)("h5",{id:"orc-parser-timeanddims-parsespec"},(0,i.yg)("inlineCode",{parentName:"h5"},"orc")," parser, ",(0,i.yg)("inlineCode",{parentName:"h5"},"timeAndDims")," parseSpec"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'{\n  "type": "index_hadoop",\n  "spec": {\n    "ioConfig": {\n      "type": "hadoop",\n      "inputSpec": {\n        "type": "static",\n        "inputFormat": "org.apache.orc.mapreduce.OrcInputFormat",\n        "paths": "path/to/file.orc"\n      },\n      ...\n    },\n    "dataSchema": {\n      "dataSource": "example",\n      "parser": {\n        "type": "orc",\n        "parseSpec": {\n          "format": "timeAndDims",\n          "timestampSpec": {\n            "column": "timestamp",\n            "format": "auto"\n          },\n          "dimensionsSpec": {\n            "dimensions": [\n              "dim1",\n              "dim2",\n              "dim3",\n              "listDim"\n            ],\n            "dimensionExclusions": [],\n            "spatialDimensions": []\n          }\n        }\n      },\n      ...\n    },\n    "tuningConfig": <hadoop-tuning-config>\n  }\n}\n\n')),(0,i.yg)("h3",{id:"parquet-hadoop-parser"},"Parquet Hadoop Parser"),(0,i.yg)("admonition",{type:"info"},(0,i.yg)("p",{parentName:"admonition"}," You need to include the ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/development/extensions-core/parquet"},(0,i.yg)("inlineCode",{parentName:"a"},"druid-parquet-extensions"))," as an extension to use the Parquet Hadoop Parser.")),(0,i.yg)("p",null,"The Parquet Hadoop parser is for ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/ingestion/hadoop"},"Hadoop batch ingestion")," and parses Parquet files directly.\nThe ",(0,i.yg)("inlineCode",{parentName:"p"},"inputFormat")," of ",(0,i.yg)("inlineCode",{parentName:"p"},"inputSpec")," in ",(0,i.yg)("inlineCode",{parentName:"p"},"ioConfig")," must be set to ",(0,i.yg)("inlineCode",{parentName:"p"},"org.apache.druid.data.input.parquet.DruidParquetInputFormat"),"."),(0,i.yg)("p",null,"The Parquet Hadoop Parser supports auto field discovery and flattening if provided with a\n",(0,i.yg)("a",{parentName:"p",href:"#flattenspec"},(0,i.yg)("inlineCode",{parentName:"a"},"flattenSpec"))," with the ",(0,i.yg)("inlineCode",{parentName:"p"},"parquet")," ",(0,i.yg)("inlineCode",{parentName:"p"},"parseSpec"),". Parquet nested list and map\n",(0,i.yg)("a",{parentName:"p",href:"https://github.com/apache/parquet-format/blob/master/LogicalTypes.md"},"logical types")," ",(0,i.yg)("em",{parentName:"p"},"should")," operate correctly with\nJSON path expressions for all supported types."),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"type"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"Set value to ",(0,i.yg)("inlineCode",{parentName:"td"},"parquet"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"parseSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the timestamp and dimensions of the data, and optionally, a flatten spec. Valid parseSpec formats are ",(0,i.yg)("inlineCode",{parentName:"td"},"timeAndDims")," and ",(0,i.yg)("inlineCode",{parentName:"td"},"parquet"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"binaryAsString"),(0,i.yg)("td",{parentName:"tr",align:null},"Boolean"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies if the bytes parquet column which is not logically marked as a string or enum type should be treated as a UTF-8 encoded string."),(0,i.yg)("td",{parentName:"tr",align:null},"no(default = false)")))),(0,i.yg)("p",null,"When the time dimension is a ",(0,i.yg)("a",{parentName:"p",href:"https://github.com/apache/parquet-format/blob/master/LogicalTypes.md"},"DateType column"),",\na format should not be supplied. When the format is UTF8 (String), either ",(0,i.yg)("inlineCode",{parentName:"p"},"auto")," or a explicitly defined\n",(0,i.yg)("a",{parentName:"p",href:"http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat"},"format")," is required."),(0,i.yg)("h4",{id:"parquet-hadoop-parser-vs-parquet-avro-hadoop-parser"},"Parquet Hadoop Parser vs Parquet Avro Hadoop Parser"),(0,i.yg)("p",null,"Both parsers read from Parquet files, but slightly differently. The main\ndifferences are:"),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},"The Parquet Hadoop Parser uses a simple conversion while the Parquet Avro Hadoop Parser\nconverts Parquet data into avro records first with the ",(0,i.yg)("inlineCode",{parentName:"li"},"parquet-avro")," library and then\nparses avro data using the ",(0,i.yg)("inlineCode",{parentName:"li"},"druid-avro-extensions")," module to ingest into Druid."),(0,i.yg)("li",{parentName:"ul"},"The Parquet Hadoop Parser sets a hadoop job property\n",(0,i.yg)("inlineCode",{parentName:"li"},"parquet.avro.add-list-element-records")," to ",(0,i.yg)("inlineCode",{parentName:"li"},"false")," (which normally defaults to ",(0,i.yg)("inlineCode",{parentName:"li"},"true"),"), in order to 'unwrap' primitive\nlist elements into multi-value dimensions."),(0,i.yg)("li",{parentName:"ul"},"The Parquet Hadoop Parser supports ",(0,i.yg)("inlineCode",{parentName:"li"},"int96")," Parquet values, while the Parquet Avro Hadoop Parser does not.\nThere may also be some subtle differences in the behavior of JSON path expression evaluation of ",(0,i.yg)("inlineCode",{parentName:"li"},"flattenSpec"),".")),(0,i.yg)("p",null,"Based on those differences, we suggest using the Parquet Hadoop Parser over the Parquet Avro Hadoop Parser\nto allow ingesting data beyond the schema constraints of Avro conversion.\nHowever, the Parquet Avro Hadoop Parser was the original basis for supporting the Parquet format, and as such it is a bit more mature."),(0,i.yg)("h4",{id:"examples-1"},"Examples"),(0,i.yg)("h5",{id:"parquet-parser-parquet-parsespec"},(0,i.yg)("inlineCode",{parentName:"h5"},"parquet")," parser, ",(0,i.yg)("inlineCode",{parentName:"h5"},"parquet")," parseSpec"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'{\n  "type": "index_hadoop",\n  "spec": {\n    "ioConfig": {\n      "type": "hadoop",\n      "inputSpec": {\n        "type": "static",\n        "inputFormat": "org.apache.druid.data.input.parquet.DruidParquetInputFormat",\n        "paths": "path/to/file.parquet"\n      },\n      ...\n    },\n    "dataSchema": {\n      "dataSource": "example",\n      "parser": {\n        "type": "parquet",\n        "parseSpec": {\n          "format": "parquet",\n          "flattenSpec": {\n            "useFieldDiscovery": true,\n            "fields": [\n              {\n                "type": "path",\n                "name": "nestedDim",\n                "expr": "$.nestedData.dim1"\n              },\n              {\n                "type": "path",\n                "name": "listDimFirstItem",\n                "expr": "$.listDim[1]"\n              }\n            ]\n          },\n          "timestampSpec": {\n            "column": "timestamp",\n            "format": "auto"\n          },\n          "dimensionsSpec": {\n            "dimensions": [],\n            "dimensionExclusions": [],\n            "spatialDimensions": []\n          }\n        }\n      },\n      ...\n    },\n    "tuningConfig": <hadoop-tuning-config>\n    }\n  }\n}\n')),(0,i.yg)("h5",{id:"parquet-parser-timeanddims-parsespec"},(0,i.yg)("inlineCode",{parentName:"h5"},"parquet")," parser, ",(0,i.yg)("inlineCode",{parentName:"h5"},"timeAndDims")," parseSpec"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'{\n  "type": "index_hadoop",\n  "spec": {\n    "ioConfig": {\n      "type": "hadoop",\n      "inputSpec": {\n        "type": "static",\n        "inputFormat": "org.apache.druid.data.input.parquet.DruidParquetInputFormat",\n        "paths": "path/to/file.parquet"\n      },\n      ...\n    },\n    "dataSchema": {\n      "dataSource": "example",\n      "parser": {\n        "type": "parquet",\n        "parseSpec": {\n          "format": "timeAndDims",\n          "timestampSpec": {\n            "column": "timestamp",\n            "format": "auto"\n          },\n          "dimensionsSpec": {\n            "dimensions": [\n              "dim1",\n              "dim2",\n              "dim3",\n              "listDim"\n            ],\n            "dimensionExclusions": [],\n            "spatialDimensions": []\n          }\n        }\n      },\n      ...\n    },\n    "tuningConfig": <hadoop-tuning-config>\n  }\n}\n\n')),(0,i.yg)("h3",{id:"parquet-avro-hadoop-parser"},"Parquet Avro Hadoop Parser"),(0,i.yg)("admonition",{type:"info"},(0,i.yg)("p",{parentName:"admonition"}," Consider using the ",(0,i.yg)("a",{parentName:"p",href:"#parquet-hadoop-parser"},"Parquet Hadoop Parser")," over this parser to ingest\nParquet files. See ",(0,i.yg)("a",{parentName:"p",href:"#parquet-hadoop-parser-vs-parquet-avro-hadoop-parser"},"Parquet Hadoop Parser vs Parquet Avro Hadoop Parser"),"\nfor the differences between those parsers.")),(0,i.yg)("admonition",{type:"info"},(0,i.yg)("p",{parentName:"admonition"}," You need to include both the ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/development/extensions-core/parquet"},(0,i.yg)("inlineCode",{parentName:"a"},"druid-parquet-extensions")),"\n","[",(0,i.yg)("inlineCode",{parentName:"p"},"druid-avro-extensions"),"]"," as extensions to use the Parquet Avro Hadoop Parser.")),(0,i.yg)("p",null,"The Parquet Avro Hadoop Parser is for ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/ingestion/hadoop"},"Hadoop batch ingestion"),".\nThis parser first converts the Parquet data into Avro records, and then parses them to ingest into Druid.\nThe ",(0,i.yg)("inlineCode",{parentName:"p"},"inputFormat")," of ",(0,i.yg)("inlineCode",{parentName:"p"},"inputSpec")," in ",(0,i.yg)("inlineCode",{parentName:"p"},"ioConfig")," must be set to ",(0,i.yg)("inlineCode",{parentName:"p"},"org.apache.druid.data.input.parquet.DruidParquetAvroInputFormat"),"."),(0,i.yg)("p",null,"The Parquet Avro Hadoop Parser supports auto field discovery and flattening if provided with a\n",(0,i.yg)("a",{parentName:"p",href:"#flattenspec"},(0,i.yg)("inlineCode",{parentName:"a"},"flattenSpec"))," with the ",(0,i.yg)("inlineCode",{parentName:"p"},"avro")," ",(0,i.yg)("inlineCode",{parentName:"p"},"parseSpec"),". Parquet nested list and map\n",(0,i.yg)("a",{parentName:"p",href:"https://github.com/apache/parquet-format/blob/master/LogicalTypes.md"},"logical types")," ",(0,i.yg)("em",{parentName:"p"},"should")," operate correctly with\nJSON path expressions for all supported types. This parser sets a hadoop job property\n",(0,i.yg)("inlineCode",{parentName:"p"},"parquet.avro.add-list-element-records")," to ",(0,i.yg)("inlineCode",{parentName:"p"},"false")," (which normally defaults to ",(0,i.yg)("inlineCode",{parentName:"p"},"true"),"), in order to 'unwrap' primitive\nlist elements into multi-value dimensions."),(0,i.yg)("p",null,"Note that the ",(0,i.yg)("inlineCode",{parentName:"p"},"int96")," Parquet value type is not supported with this parser."),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"type"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"Set value to ",(0,i.yg)("inlineCode",{parentName:"td"},"parquet-avro"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"parseSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the timestamp and dimensions of the data, and optionally, a flatten spec. Should be ",(0,i.yg)("inlineCode",{parentName:"td"},"avro"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"binaryAsString"),(0,i.yg)("td",{parentName:"tr",align:null},"Boolean"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies if the bytes parquet column which is not logically marked as a string or enum type should be treated as a UTF-8 encoded string."),(0,i.yg)("td",{parentName:"tr",align:null},"no(default = false)")))),(0,i.yg)("p",null,"When the time dimension is a ",(0,i.yg)("a",{parentName:"p",href:"https://github.com/apache/parquet-format/blob/master/LogicalTypes.md"},"DateType column"),",\na format should not be supplied. When the format is UTF8 (String), either ",(0,i.yg)("inlineCode",{parentName:"p"},"auto")," or\nan explicitly defined ",(0,i.yg)("a",{parentName:"p",href:"http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat"},"format")," is required."),(0,i.yg)("h4",{id:"example-1"},"Example"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'{\n  "type": "index_hadoop",\n  "spec": {\n    "ioConfig": {\n      "type": "hadoop",\n      "inputSpec": {\n        "type": "static",\n        "inputFormat": "org.apache.druid.data.input.parquet.DruidParquetAvroInputFormat",\n        "paths": "path/to/file.parquet"\n      },\n      ...\n    },\n    "dataSchema": {\n      "dataSource": "example",\n      "parser": {\n        "type": "parquet-avro",\n        "parseSpec": {\n          "format": "avro",\n          "flattenSpec": {\n            "useFieldDiscovery": true,\n            "fields": [\n              {\n                "type": "path",\n                "name": "nestedDim",\n                "expr": "$.nestedData.dim1"\n              },\n              {\n                "type": "path",\n                "name": "listDimFirstItem",\n                "expr": "$.listDim[1]"\n              }\n            ]\n          },\n          "timestampSpec": {\n            "column": "timestamp",\n            "format": "auto"\n          },\n          "dimensionsSpec": {\n            "dimensions": [],\n            "dimensionExclusions": [],\n            "spatialDimensions": []\n          }\n        }\n      },\n      ...\n    },\n    "tuningConfig": <hadoop-tuning-config>\n    }\n  }\n}\n')),(0,i.yg)("h3",{id:"avro-stream-parser"},"Avro Stream Parser"),(0,i.yg)("admonition",{type:"info"},(0,i.yg)("p",{parentName:"admonition"}," You need to include the ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/development/extensions-core/avro"},(0,i.yg)("inlineCode",{parentName:"a"},"druid-avro-extensions"))," as an extension to use the Avro Stream Parser.")),(0,i.yg)("admonition",{type:"info"},(0,i.yg)("p",{parentName:"admonition"}," See the ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/development/extensions-core/avro#avro-types"},"Avro Types")," section for how Avro types are handled in Druid")),(0,i.yg)("p",null,"This parser is for ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/ingestion/#streaming"},"stream ingestion")," and reads Avro data from a stream directly."),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"type"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"Set value to ",(0,i.yg)("inlineCode",{parentName:"td"},"avro_stream"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"no")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"avroBytesDecoder"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies ","[",(0,i.yg)("inlineCode",{parentName:"td"},"avroBytesDecoder"),"]","(#Avro Bytes Decoder) to decode bytes to Avro record."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"parseSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},'Specifies the timestamp and dimensions of the data. Should be an "avro" parseSpec.'),(0,i.yg)("td",{parentName:"tr",align:null},"yes")))),(0,i.yg)("p",null,"An Avro parseSpec can contain a ",(0,i.yg)("a",{parentName:"p",href:"#flattenspec"},(0,i.yg)("inlineCode",{parentName:"a"},"flattenSpec")),' using either the "root" or "path"\nfield types, which can be used to read nested Avro records. The "jq" and "tree" field type is not currently supported for Avro.'),(0,i.yg)("p",null,"For example, using Avro stream parser with schema repo Avro bytes decoder:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'"parser" : {\n  "type" : "avro_stream",\n  "avroBytesDecoder" : {\n    "type" : "schema_repo",\n    "subjectAndIdConverter" : {\n      "type" : "avro_1124",\n      "topic" : "${YOUR_TOPIC}"\n    },\n    "schemaRepository" : {\n      "type" : "avro_1124_rest_client",\n      "url" : "${YOUR_SCHEMA_REPO_END_POINT}",\n    }\n  },\n  "parseSpec" : {\n    "format": "avro",\n    "timestampSpec": <standard timestampSpec>,\n    "dimensionsSpec": <standard dimensionsSpec>,\n    "flattenSpec": <optional>\n  }\n}\n')),(0,i.yg)("h3",{id:"protobuf-parser"},"Protobuf Parser"),(0,i.yg)("admonition",{type:"info"},(0,i.yg)("p",{parentName:"admonition"}," You need to include the ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/development/extensions-core/protobuf"},(0,i.yg)("inlineCode",{parentName:"a"},"druid-protobuf-extensions"))," as an extension to use the Protobuf Parser.")),(0,i.yg)("p",null,"This parser is for ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/ingestion/#streaming"},"stream ingestion")," and reads Protocol buffer data from a stream directly."),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"type"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"Set value to ",(0,i.yg)("inlineCode",{parentName:"td"},"protobuf"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"protoBytesDecoder")),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies how to decode bytes to Protobuf record."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"parseSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the timestamp and dimensions of the data.  The format must be JSON. See ",(0,i.yg)("a",{parentName:"td",href:"#json-parsespec"},"JSON ParseSpec")," for more configuration options. Note that ",(0,i.yg)("inlineCode",{parentName:"td"},"timeAndDims")," ",(0,i.yg)("inlineCode",{parentName:"td"},"parseSpec")," is no longer supported."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")))),(0,i.yg)("p",null,"Sample spec:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'"parser": {\n  "type": "protobuf",\n  "protoBytesDecoder": {\n    "type": "file",\n    "descriptor": "file:///tmp/metrics.desc",\n    "protoMessageType": "Metrics"\n  },\n  "parseSpec": {\n    "format": "json",\n    "timestampSpec": {\n      "column": "timestamp",\n      "format": "auto"\n    },\n    "dimensionsSpec": {\n      "dimensions": [\n        "unit",\n        "http_method",\n        "http_code",\n        "page",\n        "metricType",\n        "server"\n      ],\n      "dimensionExclusions": [\n        "timestamp",\n        "value"\n      ]\n    }\n  }\n}\n')),(0,i.yg)("p",null,"See the ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/development/extensions-core/protobuf"},"extension description")," for\nmore details and examples."),(0,i.yg)("h4",{id:"protobuf-bytes-decoder"},"Protobuf Bytes Decoder"),(0,i.yg)("p",null,"If ",(0,i.yg)("inlineCode",{parentName:"p"},"type")," is not included, the ",(0,i.yg)("inlineCode",{parentName:"p"},"protoBytesDecoder")," defaults to ",(0,i.yg)("inlineCode",{parentName:"p"},"schema_registry"),"."),(0,i.yg)("h5",{id:"file-based-protobuf-bytes-decoder"},"File-based Protobuf Bytes Decoder"),(0,i.yg)("p",null,"This Protobuf bytes decoder first read a descriptor file, and then parse it to get schema used to decode the Protobuf record from bytes."),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"type"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"Set value to ",(0,i.yg)("inlineCode",{parentName:"td"},"file"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"descriptor"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"Protobuf descriptor file name in the classpath or URL."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"protoMessageType"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"Protobuf message type in the descriptor.  Both short name and fully qualified name are accepted. The parser uses the first message type found in the descriptor if not specified."),(0,i.yg)("td",{parentName:"tr",align:null},"no")))),(0,i.yg)("p",null,"Sample spec:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'"protoBytesDecoder": {\n  "type": "file",\n  "descriptor": "file:///tmp/metrics.desc",\n  "protoMessageType": "Metrics"\n}\n')),(0,i.yg)("h4",{id:"inline-descriptor-protobuf-bytes-decoder"},"Inline Descriptor Protobuf Bytes Decoder"),(0,i.yg)("p",null,"This Protobuf bytes decoder allows the user to provide the contents of a Protobuf descriptor file inline, encoded as a Base64 string, and then parse it to get schema used to decode the Protobuf record from bytes."),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"type"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"Set value to ",(0,i.yg)("inlineCode",{parentName:"td"},"inline"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"descriptorString"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"A compiled Protobuf descriptor, encoded as a Base64 string."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"protoMessageType"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"Protobuf message type in the descriptor.  Both short name and fully qualified name are accepted. The parser uses the first message type found in the descriptor if not specified."),(0,i.yg)("td",{parentName:"tr",align:null},"no")))),(0,i.yg)("p",null,"Sample spec:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'"protoBytesDecoder": {\n  "type": "inline",\n  "descriptorString": <Contents of a Protobuf descriptor file encoded as Base64 string>,\n  "protoMessageType": "Metrics"\n}\n')),(0,i.yg)("h5",{id:"confluent-schema-registry-based-protobuf-bytes-decoder"},"Confluent Schema Registry-based Protobuf Bytes Decoder"),(0,i.yg)("p",null,"This Protobuf bytes decoder first extracts a unique ",(0,i.yg)("inlineCode",{parentName:"p"},"id")," from input message bytes, and then uses it to look up the schema in the Schema Registry used to decode the Avro record from bytes.\nFor details, see the Schema Registry ",(0,i.yg)("a",{parentName:"p",href:"http://docs.confluent.io/current/schema-registry/docs/"},"documentation")," and ",(0,i.yg)("a",{parentName:"p",href:"https://github.com/confluentinc/schema-registry"},"repository"),"."),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"type"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"Set value to ",(0,i.yg)("inlineCode",{parentName:"td"},"schema_registry"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"url"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the URL endpoint of the Schema Registry."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"capacity"),(0,i.yg)("td",{parentName:"tr",align:null},"Integer"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the max size of the cache (default = Integer.MAX_VALUE)."),(0,i.yg)("td",{parentName:"tr",align:null},"no")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"urls"),(0,i.yg)("td",{parentName:"tr",align:null},"Array<String",">"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the URL endpoints of the multiple Schema Registry instances."),(0,i.yg)("td",{parentName:"tr",align:null},"yes (if ",(0,i.yg)("inlineCode",{parentName:"td"},"url")," is not provided)")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"config"),(0,i.yg)("td",{parentName:"tr",align:null},"Json"),(0,i.yg)("td",{parentName:"tr",align:null},"To send additional configurations, configured for Schema Registry. This can be supplied via a ",(0,i.yg)("a",{parentName:"td",href:"/docs/latest/operations/dynamic-config-provider"},"DynamicConfigProvider"),"."),(0,i.yg)("td",{parentName:"tr",align:null},"no")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"headers"),(0,i.yg)("td",{parentName:"tr",align:null},"Json"),(0,i.yg)("td",{parentName:"tr",align:null},"To send headers to the Schema Registry.  This can be supplied via a ",(0,i.yg)("a",{parentName:"td",href:"/docs/latest/operations/dynamic-config-provider"},"DynamicConfigProvider")),(0,i.yg)("td",{parentName:"tr",align:null},"no")))),(0,i.yg)("p",null,"For a single schema registry instance, use Field ",(0,i.yg)("inlineCode",{parentName:"p"},"url")," or ",(0,i.yg)("inlineCode",{parentName:"p"},"urls")," for multi instances."),(0,i.yg)("p",null,"Single Instance:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'...\n"protoBytesDecoder": {\n  "url": <schema-registry-url>,\n  "type": "schema_registry"\n}\n...\n')),(0,i.yg)("p",null,"Multiple Instances:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'...\n"protoBytesDecoder": {\n  "urls": [<schema-registry-url-1>, <schema-registry-url-2>, ...],\n  "type": "schema_registry",\n  "capacity": 100,\n  "config" : {\n       "basic.auth.credentials.source": "USER_INFO",\n       "basic.auth.user.info": "fred:letmein",\n       "schema.registry.ssl.truststore.location": "/some/secrets/kafka.client.truststore.jks",\n       "schema.registry.ssl.truststore.password": "<password>",\n       "schema.registry.ssl.keystore.location": "/some/secrets/kafka.client.keystore.jks",\n       "schema.registry.ssl.keystore.password": "<password>",\n       "schema.registry.ssl.key.password": "<password>",\n         ... \n  },\n  "headers": {\n      "traceID" : "b29c5de2-0db4-490b-b421",\n      "timeStamp" : "1577191871865",\n      "druid.dynamic.config.provider":{\n           "type":"mapString", \n           "config":{\n                "registry.header.prop.1":"value.1", \n                "registry.header.prop.2":"value.2"\n                }\n           }\n      ...\n  }\n}\n...\n')),(0,i.yg)("h2",{id:"parsespec"},"ParseSpec"),(0,i.yg)("admonition",{type:"info"},(0,i.yg)("p",{parentName:"admonition"}," The Parser is deprecated for ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/ingestion/native-batch"},"native batch tasks"),", ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/ingestion/kafka-ingestion"},"Kafka indexing service"),",\nand ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/ingestion/kinesis-ingestion"},"Kinesis indexing service"),".\nConsider using the ",(0,i.yg)("a",{parentName:"p",href:"#input-format"},"input format")," instead for these types of ingestion.")),(0,i.yg)("p",null,"ParseSpecs serve two purposes:"),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},"The String Parser use them to determine the format (i.e., JSON, CSV, TSV) of incoming rows."),(0,i.yg)("li",{parentName:"ul"},"All Parsers use them to determine the timestamp and dimensions of incoming rows.")),(0,i.yg)("p",null,"If ",(0,i.yg)("inlineCode",{parentName:"p"},"format")," is not included, the parseSpec defaults to ",(0,i.yg)("inlineCode",{parentName:"p"},"tsv"),"."),(0,i.yg)("h3",{id:"json-parsespec"},"JSON ParseSpec"),(0,i.yg)("p",null,"Use this with the String Parser to load JSON."),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"format"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"json")),(0,i.yg)("td",{parentName:"tr",align:null},"no")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"timestampSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the column and format of the timestamp."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"dimensionsSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the dimensions of the data."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"flattenSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies flattening configuration for nested JSON data. See ",(0,i.yg)("a",{parentName:"td",href:"#flattenspec"},(0,i.yg)("inlineCode",{parentName:"a"},"flattenSpec"))," for more info."),(0,i.yg)("td",{parentName:"tr",align:null},"no")))),(0,i.yg)("p",null,"Sample spec:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'"parseSpec": {\n  "format" : "json",\n  "timestampSpec" : {\n    "column" : "timestamp"\n  },\n  "dimensionSpec" : {\n    "dimensions" : ["page","language","user","unpatrolled","newPage","robot","anonymous","namespace","continent","country","region","city"]\n  }\n}\n')),(0,i.yg)("h3",{id:"json-lowercase-parsespec"},"JSON Lowercase ParseSpec"),(0,i.yg)("admonition",{type:"info"},(0,i.yg)("p",{parentName:"admonition"}," The ",(0,i.yg)("em",{parentName:"p"},"jsonLowercase")," parser is deprecated and may be removed in a future version of Druid.")),(0,i.yg)("p",null,"This is a special variation of the JSON ParseSpec that lower cases all the column names in the incoming JSON data. This parseSpec is required if you are updating to Druid 0.7.x from Druid 0.6.x, are directly ingesting JSON with mixed case column names, do not have any ETL in place to lower case those column names, and would like to make queries that include the data you created using 0.6.x and 0.7.x."),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"format"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"jsonLowercase")),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"timestampSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the column and format of the timestamp."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"dimensionsSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the dimensions of the data."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")))),(0,i.yg)("h3",{id:"csv-parsespec"},"CSV ParseSpec"),(0,i.yg)("p",null,"Use this with the String Parser to load CSV. Strings are parsed using the com.opencsv library."),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"format"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"csv")),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"timestampSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the column and format of the timestamp."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"dimensionsSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the dimensions of the data."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"listDelimiter"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"A custom delimiter for multi-value dimensions."),(0,i.yg)("td",{parentName:"tr",align:null},"no (default = ctrl+A)")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"columns"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON array"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the columns of the data."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")))),(0,i.yg)("p",null,"Sample spec:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'"parseSpec": {\n  "format" : "csv",\n  "timestampSpec" : {\n    "column" : "timestamp"\n  },\n  "columns" : ["timestamp","page","language","user","unpatrolled","newPage","robot","anonymous","namespace","continent","country","region","city","added","deleted","delta"],\n  "dimensionsSpec" : {\n    "dimensions" : ["page","language","user","unpatrolled","newPage","robot","anonymous","namespace","continent","country","region","city"]\n  }\n}\n')),(0,i.yg)("h4",{id:"csv-index-tasks"},"CSV Index Tasks"),(0,i.yg)("p",null,"If your input files contain a header, the ",(0,i.yg)("inlineCode",{parentName:"p"},"columns")," field is optional and you don't need to set.\nInstead, you can set the ",(0,i.yg)("inlineCode",{parentName:"p"},"hasHeaderRow")," field to true, which makes Druid automatically extract the column information from the header.\nOtherwise, you must set the ",(0,i.yg)("inlineCode",{parentName:"p"},"columns")," field and ensure that field must match the columns of your input data in the same order."),(0,i.yg)("p",null,"Also, you can skip some header rows by setting ",(0,i.yg)("inlineCode",{parentName:"p"},"skipHeaderRows")," in your parseSpec. If both ",(0,i.yg)("inlineCode",{parentName:"p"},"skipHeaderRows")," and ",(0,i.yg)("inlineCode",{parentName:"p"},"hasHeaderRow")," options are set,\n",(0,i.yg)("inlineCode",{parentName:"p"},"skipHeaderRows")," is first applied. For example, if you set ",(0,i.yg)("inlineCode",{parentName:"p"},"skipHeaderRows")," to 2 and ",(0,i.yg)("inlineCode",{parentName:"p"},"hasHeaderRow")," to true, Druid will\nskip the first two lines and then extract column information from the third line."),(0,i.yg)("p",null,"Note that ",(0,i.yg)("inlineCode",{parentName:"p"},"hasHeaderRow")," and ",(0,i.yg)("inlineCode",{parentName:"p"},"skipHeaderRows")," are effective only for non-Hadoop batch index tasks. Other types of index\ntasks will fail with an exception."),(0,i.yg)("h4",{id:"other-csv-ingestion-tasks"},"Other CSV Ingestion Tasks"),(0,i.yg)("p",null,"The ",(0,i.yg)("inlineCode",{parentName:"p"},"columns")," field must be included and and ensure that the order of the fields matches the columns of your input data in the same order."),(0,i.yg)("h3",{id:"tsv--delimited-parsespec"},"TSV / Delimited ParseSpec"),(0,i.yg)("p",null,"Use this with the String Parser to load any delimited text that does not require special escaping. By default,\nthe delimiter is a tab, so this will load TSV."),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"format"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"tsv")),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"timestampSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the column and format of the timestamp."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"dimensionsSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the dimensions of the data."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"delimiter"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"A custom delimiter for data values."),(0,i.yg)("td",{parentName:"tr",align:null},"no (default = \\t)")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"listDelimiter"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},"A custom delimiter for multi-value dimensions."),(0,i.yg)("td",{parentName:"tr",align:null},"no (default = ctrl+A)")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"columns"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON String array"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the columns of the data."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")))),(0,i.yg)("p",null,"Sample spec:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'"parseSpec": {\n  "format" : "tsv",\n  "timestampSpec" : {\n    "column" : "timestamp"\n  },\n  "columns" : ["timestamp","page","language","user","unpatrolled","newPage","robot","anonymous","namespace","continent","country","region","city","added","deleted","delta"],\n  "delimiter":"|",\n  "dimensionsSpec" : {\n    "dimensions" : ["page","language","user","unpatrolled","newPage","robot","anonymous","namespace","continent","country","region","city"]\n  }\n}\n')),(0,i.yg)("p",null,"Be sure to change the ",(0,i.yg)("inlineCode",{parentName:"p"},"delimiter")," to the appropriate delimiter for your data. Like CSV, you must specify the columns and which subset of the columns you want indexed."),(0,i.yg)("h4",{id:"tsv-delimited-index-tasks"},"TSV (Delimited) Index Tasks"),(0,i.yg)("p",null,"If your input files contain a header, the ",(0,i.yg)("inlineCode",{parentName:"p"},"columns")," field is optional and doesn't need to be set.\nInstead, you can set the ",(0,i.yg)("inlineCode",{parentName:"p"},"hasHeaderRow")," field to true, which makes Druid automatically extract the column information from the header.\nOtherwise, you must set the ",(0,i.yg)("inlineCode",{parentName:"p"},"columns")," field and ensure that field must match the columns of your input data in the same order."),(0,i.yg)("p",null,"Also, you can skip some header rows by setting ",(0,i.yg)("inlineCode",{parentName:"p"},"skipHeaderRows")," in your parseSpec. If both ",(0,i.yg)("inlineCode",{parentName:"p"},"skipHeaderRows")," and ",(0,i.yg)("inlineCode",{parentName:"p"},"hasHeaderRow")," options are set,\n",(0,i.yg)("inlineCode",{parentName:"p"},"skipHeaderRows")," is first applied. For example, if you set ",(0,i.yg)("inlineCode",{parentName:"p"},"skipHeaderRows")," to 2 and ",(0,i.yg)("inlineCode",{parentName:"p"},"hasHeaderRow")," to true, Druid will\nskip the first two lines and then extract column information from the third line."),(0,i.yg)("p",null,"Note that ",(0,i.yg)("inlineCode",{parentName:"p"},"hasHeaderRow")," and ",(0,i.yg)("inlineCode",{parentName:"p"},"skipHeaderRows")," are effective only for non-Hadoop batch index tasks. Other types of index\ntasks will fail with an exception."),(0,i.yg)("h4",{id:"other-tsv-delimited-ingestion-tasks"},"Other TSV (Delimited) Ingestion Tasks"),(0,i.yg)("p",null,"The ",(0,i.yg)("inlineCode",{parentName:"p"},"columns")," field must be included and and ensure that the order of the fields matches the columns of your input data in the same order."),(0,i.yg)("h3",{id:"regex-parsespec"},"Regex ParseSpec"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'"parseSpec":{\n  "format" : "regex",\n  "timestampSpec" : {\n    "column" : "timestamp"\n  },\n  "dimensionsSpec" : {\n    "dimensions" : [<your_list_of_dimensions>]\n  },\n  "columns" : [<your_columns_here>],\n  "pattern" : <regex pattern for partitioning data>\n}\n')),(0,i.yg)("p",null,"The ",(0,i.yg)("inlineCode",{parentName:"p"},"columns"),' field must match the columns of your regex matching groups in the same order. If columns are not provided, default\ncolumns names ("column_1", "column2", ... "column_n") will be assigned. Ensure that your column names include all your dimensions.'),(0,i.yg)("h3",{id:"javascript-parsespec"},"JavaScript ParseSpec"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-json"},'"parseSpec":{\n  "format" : "javascript",\n  "timestampSpec" : {\n    "column" : "timestamp"\n  },\n  "dimensionsSpec" : {\n    "dimensions" : ["page","language","user","unpatrolled","newPage","robot","anonymous","namespace","continent","country","region","city"]\n  },\n  "function" : "function(str) { var parts = str.split(\\"-\\"); return { one: parts[0], two: parts[1] } }"\n}\n')),(0,i.yg)("p",null,"Note with the JavaScript parser that data must be fully parsed and returned as a ",(0,i.yg)("inlineCode",{parentName:"p"},"{key:value}")," format in the JS logic.\nThis means any flattening or parsing multi-dimensional values must be done here."),(0,i.yg)("admonition",{type:"info"},(0,i.yg)("p",{parentName:"admonition"}," JavaScript-based functionality is disabled by default. Please refer to the Druid ",(0,i.yg)("a",{parentName:"p",href:"/docs/latest/development/javascript"},"JavaScript programming guide")," for guidelines about using Druid's JavaScript functionality, including instructions on how to enable it.")),(0,i.yg)("h3",{id:"timeanddims-parsespec"},"TimeAndDims ParseSpec"),(0,i.yg)("p",null,"Use this with non-String Parsers to provide them with timestamp and dimensions information. Non-String Parsers\nhandle all formatting decisions on their own, without using the ParseSpec."),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"format"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"timeAndDims")),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"timestampSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the column and format of the timestamp."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"dimensionsSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the dimensions of the data."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")))),(0,i.yg)("h3",{id:"orc-parsespec"},"Orc ParseSpec"),(0,i.yg)("p",null,"Use this with the Hadoop ORC Parser to load ORC files."),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"format"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"orc")),(0,i.yg)("td",{parentName:"tr",align:null},"no")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"timestampSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the column and format of the timestamp."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"dimensionsSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the dimensions of the data."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"flattenSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies flattening configuration for nested JSON data. See ",(0,i.yg)("a",{parentName:"td",href:"#flattenspec"},(0,i.yg)("inlineCode",{parentName:"a"},"flattenSpec"))," for more info."),(0,i.yg)("td",{parentName:"tr",align:null},"no")))),(0,i.yg)("h3",{id:"parquet-parsespec"},"Parquet ParseSpec"),(0,i.yg)("p",null,"Use this with the Hadoop Parquet Parser to load Parquet files."),(0,i.yg)("table",null,(0,i.yg)("thead",{parentName:"table"},(0,i.yg)("tr",{parentName:"thead"},(0,i.yg)("th",{parentName:"tr",align:null},"Field"),(0,i.yg)("th",{parentName:"tr",align:null},"Type"),(0,i.yg)("th",{parentName:"tr",align:null},"Description"),(0,i.yg)("th",{parentName:"tr",align:null},"Required"))),(0,i.yg)("tbody",{parentName:"table"},(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"format"),(0,i.yg)("td",{parentName:"tr",align:null},"String"),(0,i.yg)("td",{parentName:"tr",align:null},(0,i.yg)("inlineCode",{parentName:"td"},"parquet")),(0,i.yg)("td",{parentName:"tr",align:null},"no")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"timestampSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the column and format of the timestamp."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"dimensionsSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies the dimensions of the data."),(0,i.yg)("td",{parentName:"tr",align:null},"yes")),(0,i.yg)("tr",{parentName:"tbody"},(0,i.yg)("td",{parentName:"tr",align:null},"flattenSpec"),(0,i.yg)("td",{parentName:"tr",align:null},"JSON Object"),(0,i.yg)("td",{parentName:"tr",align:null},"Specifies flattening configuration for nested JSON data. See ",(0,i.yg)("a",{parentName:"td",href:"#flattenspec"},(0,i.yg)("inlineCode",{parentName:"a"},"flattenSpec"))," for more info."),(0,i.yg)("td",{parentName:"tr",align:null},"no")))))}u.isMDXComponent=!0}}]);