| "use strict";(self.webpackChunk=self.webpackChunk||[]).push([[8915],{28453:(e,n,i)=>{i.d(n,{R:()=>r,x:()=>a});var o=i(96540);const t={},s=o.createContext(t);function r(e){const n=o.useContext(s);return o.useMemo((function(){return"function"==typeof e?e(n):{...n,...e}}),[n,e])}function a(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(t):e.components||t:r(e.components),o.createElement(s.Provider,{value:n},e.children)}},91179:(e,n,i)=>{i.r(n),i.d(n,{assets:()=>c,contentTitle:()=>a,default:()=>p,frontMatter:()=>r,metadata:()=>o,toc:()=>l});const o=JSON.parse('{"id":"operations/migrate-from-firehose","title":"Migrate from firehose to input source ingestion (legacy)","description":"\x3c!--","source":"@site/docs/32.0.0/operations/migrate-from-firehose-ingestion.md","sourceDirName":"operations","slug":"/operations/migrate-from-firehose","permalink":"/docs/32.0.0/operations/migrate-from-firehose","draft":false,"unlisted":false,"tags":[],"version":"current","frontMatter":{"id":"migrate-from-firehose","title":"Migrate from firehose to input source ingestion (legacy)","sidebar_label":"Migrate from firehose"},"sidebar":"docs","previous":{"title":"Using rules to drop and retain data","permalink":"/docs/32.0.0/operations/rule-configuration"},"next":{"title":"Working with different versions of Apache Hadoop","permalink":"/docs/32.0.0/operations/other-hadoop"}}');var t=i(74848),s=i(28453);const r={id:"migrate-from-firehose",title:"Migrate from firehose to input source ingestion (legacy)",sidebar_label:"Migrate from firehose"},a=void 0,c={},l=[{value:"Migrate from firehose ingestion to an input source",id:"migrate-from-firehose-ingestion-to-an-input-source",level:2},{value:"Use the Druid console",id:"use-the-druid-console",level:3},{value:"Update your ingestion spec manually",id:"update-your-ingestion-spec-manually",level:3},{value:"Example firehose ingestion spec",id:"example-firehose-ingestion-spec",level:3},{value:"Example ingestion spec after migration",id:"example-ingestion-spec-after-migration",level:3},{value:"Learn more",id:"learn-more",level:2}];function d(e){const n={a:"a",code:"code",h2:"h2",h3:"h3",li:"li",ol:"ol",p:"p",pre:"pre",strong:"strong",ul:"ul",...(0,s.R)(),...e.components};return(0,t.jsxs)(t.Fragment,{children:[(0,t.jsx)(n.p,{children:"Apache deprecated support for Druid firehoses in version 0.17. Support for firehose ingestion was removed in version 26.0."}),"\n",(0,t.jsx)(n.p,{children:"Firehose ingestion doesn't work with newer Druid versions, so you must be using an ingestion spec with a defined input source before you upgrade."}),"\n",(0,t.jsx)(n.h2,{id:"migrate-from-firehose-ingestion-to-an-input-source",children:"Migrate from firehose ingestion to an input source"}),"\n",(0,t.jsx)(n.p,{children:"To migrate from firehose ingestion, you can use the Druid console to update your ingestion spec, or you can update it manually."}),"\n",(0,t.jsx)(n.h3,{id:"use-the-druid-console",children:"Use the Druid console"}),"\n",(0,t.jsxs)(n.p,{children:["To update your ingestion spec using the Druid console, open the console and copy your spec into the ",(0,t.jsx)(n.strong,{children:"Edit spec"})," stage of the data loader."]}),"\n",(0,t.jsxs)(n.p,{children:["Druid converts the spec into one with a defined input source. For example, it converts the ",(0,t.jsx)(n.a,{href:"#example-firehose-ingestion-spec",children:"example firehose ingestion spec"})," below into the ",(0,t.jsx)(n.a,{href:"#example-ingestion-spec-after-migration",children:"example ingestion spec after migration"}),"."]}),"\n",(0,t.jsx)(n.p,{children:"If you're unable to use the console or you have problems with the console method, the alternative is to update your ingestion spec manually."}),"\n",(0,t.jsx)(n.h3,{id:"update-your-ingestion-spec-manually",children:"Update your ingestion spec manually"}),"\n",(0,t.jsxs)(n.p,{children:["To update your ingestion spec manually, copy your existing spec into a new file. Refer to ",(0,t.jsx)(n.a,{href:"/docs/32.0.0/ingestion/native-batch-firehose",children:"Native batch ingestion with firehose (Deprecated)"})," for a description of firehose properties."]}),"\n",(0,t.jsx)(n.p,{children:"Edit the new file as follows:"}),"\n",(0,t.jsxs)(n.ol,{children:["\n",(0,t.jsxs)(n.li,{children:["In the ",(0,t.jsx)(n.code,{children:"ioConfig"})," component, replace the ",(0,t.jsx)(n.code,{children:"firehose"})," definition with an ",(0,t.jsx)(n.code,{children:"inputSource"})," definition for your chosen input source. See ",(0,t.jsx)(n.a,{href:"/docs/32.0.0/ingestion/input-sources",children:"Native batch input sources"})," for details."]}),"\n",(0,t.jsxs)(n.li,{children:["Move the ",(0,t.jsx)(n.code,{children:"timeStampSpec"})," definition from ",(0,t.jsx)(n.code,{children:"parser.parseSpec"})," to the ",(0,t.jsx)(n.code,{children:"dataSchema"})," component."]}),"\n",(0,t.jsxs)(n.li,{children:["Move the ",(0,t.jsx)(n.code,{children:"dimensionsSpec"})," definition from ",(0,t.jsx)(n.code,{children:"parser.parseSpec"})," to the ",(0,t.jsx)(n.code,{children:"dataSchema"})," component."]}),"\n",(0,t.jsxs)(n.li,{children:["Move the ",(0,t.jsx)(n.code,{children:"format"})," definition from ",(0,t.jsx)(n.code,{children:"parser.parseSpec"})," to an ",(0,t.jsx)(n.code,{children:"inputFormat"})," definition in ",(0,t.jsx)(n.code,{children:"ioConfig"}),"."]}),"\n",(0,t.jsxs)(n.li,{children:["Delete the ",(0,t.jsx)(n.code,{children:"parser"})," definition."]}),"\n",(0,t.jsxs)(n.li,{children:["Save the file.\nYou can check the format of your new ingestion file against the ",(0,t.jsx)(n.a,{href:"#example-ingestion-spec-after-migration",children:"migrated example"})," below."]}),"\n",(0,t.jsx)(n.li,{children:"Test the new ingestion spec with a temporary data source."}),"\n",(0,t.jsx)(n.li,{children:"Once you've successfully ingested sample data with the new spec, stop firehose ingestion and switch to the new spec."}),"\n"]}),"\n",(0,t.jsxs)(n.p,{children:["When the transition is complete, you can upgrade Druid to the latest version. See the ",(0,t.jsx)(n.a,{href:"https://druid.apache.org/downloads.html",children:"Druid release notes"})," for upgrade instructions."]}),"\n",(0,t.jsx)(n.h3,{id:"example-firehose-ingestion-spec",children:"Example firehose ingestion spec"}),"\n",(0,t.jsx)(n.p,{children:"An example firehose ingestion spec is as follows:"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-json",children:'{\n "type" : "index",\n "spec" : {\n "dataSchema" : {\n "dataSource" : "wikipedia",\n "metricsSpec" : [\n {\n "type" : "count",\n "name" : "count"\n },\n {\n "type" : "doubleSum",\n "name" : "added",\n "fieldName" : "added"\n },\n {\n "type" : "doubleSum",\n "name" : "deleted",\n "fieldName" : "deleted"\n },\n {\n "type" : "doubleSum",\n "name" : "delta",\n "fieldName" : "delta"\n }\n ],\n "granularitySpec" : {\n "type" : "uniform",\n "segmentGranularity" : "DAY",\n "queryGranularity" : "NONE",\n "intervals" : [ "2013-08-31/2013-09-01" ]\n },\n "parser": {\n "type": "string",\n "parseSpec": {\n "format": "json",\n "timestampSpec" : {\n "column" : "timestamp",\n "format" : "auto"\n },\n "dimensionsSpec" : {\n "dimensions": ["country", "page","language","user","unpatrolled","newPage","robot","anonymous","namespace","continent","region","city"],\n "dimensionExclusions" : []\n }\n }\n }\n },\n "ioConfig" : {\n "type" : "index",\n "firehose" : {\n "type" : "local",\n "baseDir" : "examples/indexing/",\n "filter" : "wikipedia_data.json"\n }\n },\n "tuningConfig" : {\n "type" : "index",\n "partitionsSpec": {\n "type": "single_dim",\n "partitionDimension": "country",\n "targetRowsPerSegment": 5000000\n }\n }\n }\n}\n'})}),"\n",(0,t.jsx)(n.h3,{id:"example-ingestion-spec-after-migration",children:"Example ingestion spec after migration"}),"\n",(0,t.jsxs)(n.p,{children:["The following example illustrates the result of migrating the ",(0,t.jsx)(n.a,{href:"#example-firehose-ingestion-spec",children:"example firehose ingestion spec"})," to a spec with an input source:"]}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-json",children:'{\n "type" : "index",\n "spec" : {\n "dataSchema" : {\n "dataSource" : "wikipedia",\n "timestampSpec" : {\n "column" : "timestamp",\n "format" : "auto"\n },\n "dimensionsSpec" : {\n "dimensions": ["country", "page","language","user","unpatrolled","newPage","robot","anonymous","namespace","continent","region","city"],\n "dimensionExclusions" : []\n },\n "metricsSpec" : [\n {\n "type" : "count",\n "name" : "count"\n },\n {\n "type" : "doubleSum",\n "name" : "added",\n "fieldName" : "added"\n },\n {\n "type" : "doubleSum",\n "name" : "deleted",\n "fieldName" : "deleted"\n },\n {\n "type" : "doubleSum",\n "name" : "delta",\n "fieldName" : "delta"\n }\n ],\n "granularitySpec" : {\n "type" : "uniform",\n "segmentGranularity" : "DAY",\n "queryGranularity" : "NONE",\n "intervals" : [ "2013-08-31/2013-09-01" ]\n }\n },\n "ioConfig" : {\n "type" : "index",\n "inputSource" : {\n "type" : "local",\n "baseDir" : "examples/indexing/",\n "filter" : "wikipedia_data.json"\n },\n "inputFormat": {\n "type": "json"\n }\n },\n "tuningConfig" : {\n "type" : "index",\n "partitionsSpec": {\n "type": "single_dim",\n "partitionDimension": "country",\n "targetRowsPerSegment": 5000000\n }\n }\n }\n}\n'})}),"\n",(0,t.jsx)(n.h2,{id:"learn-more",children:"Learn more"}),"\n",(0,t.jsx)(n.p,{children:"For more information, see the following pages:"}),"\n",(0,t.jsxs)(n.ul,{children:["\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.a,{href:"/docs/32.0.0/ingestion/",children:"Ingestion"}),": Overview of the Druid ingestion process."]}),"\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.a,{href:"/docs/32.0.0/ingestion/native-batch",children:"Native batch ingestion"}),": Description of the supported native batch indexing tasks."]}),"\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.a,{href:"/docs/32.0.0/ingestion/ingestion-spec",children:"Ingestion spec reference"}),": Description of the components and properties in the ingestion spec."]}),"\n"]})]})}function p(e={}){const{wrapper:n}={...(0,s.R)(),...e.components};return n?(0,t.jsx)(n,{...e,children:(0,t.jsx)(d,{...e})}):d(e)}}}]); |