| "use strict";(self.webpackChunk=self.webpackChunk||[]).push([[4297],{15680:(e,t,n)=>{n.d(t,{xA:()=>p,yg:()=>y});var o=n(96540);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function i(e,t){if(null==e)return{};var n,o,r=function(e,t){if(null==e)return{};var n,o,r={},a=Object.keys(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var g=o.createContext({}),d=function(e){var t=o.useContext(g),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},p=function(e){var t=d(e.components);return o.createElement(g.Provider,{value:t},e.children)},u="mdxType",s={inlineCode:"code",wrapper:function(e){var t=e.children;return o.createElement(o.Fragment,{},t)}},c=o.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,g=e.parentName,p=i(e,["components","mdxType","originalType","parentName"]),u=d(n),c=r,y=u["".concat(g,".").concat(c)]||u[c]||s[c]||a;return n?o.createElement(y,l(l({ref:t},p),{},{components:n})):o.createElement(y,l({ref:t},p))}));function y(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,l=new Array(a);l[0]=c;var i={};for(var g in t)hasOwnProperty.call(t,g)&&(i[g]=t[g]);i.originalType=e,i[u]="string"==typeof e?e:r,l[1]=i;for(var d=2;d<a;d++)l[d]=n[d];return o.createElement.apply(null,l)}return o.createElement.apply(null,n)}c.displayName="MDXCreateElement"},89328:(e,t,n)=>{n.r(t),n.d(t,{assets:()=>p,contentTitle:()=>g,default:()=>y,frontMatter:()=>i,metadata:()=>d,toc:()=>u});var o=n(58168),r=n(98587),a=(n(96540),n(15680)),l=["components"],i={id:"google",title:"Google Cloud Storage"},g=void 0,d={unversionedId:"development/extensions-core/google",id:"development/extensions-core/google",title:"Google Cloud Storage",description:"\x3c!--",source:"@site/docs/latest/development/extensions-core/google.md",sourceDirName:"development/extensions-core",slug:"/development/extensions-core/google",permalink:"/docs/latest/development/extensions-core/google",draft:!1,tags:[],version:"current",frontMatter:{id:"google",title:"Google Cloud Storage"}},p={},u=[{value:"Google Cloud Storage Extension",id:"google-cloud-storage-extension",level:2},{value:"Required Configuration",id:"required-configuration",level:3},{value:"Reading data from Google Cloud Storage",id:"reading-data-from-google-cloud-storage",level:3},{value:"Deep Storage",id:"deep-storage",level:3},{value:"Configuration",id:"configuration",level:4}],s={toc:u},c="wrapper";function y(e){var t=e.components,n=(0,r.A)(e,l);return(0,a.yg)(c,(0,o.A)({},s,n,{components:t,mdxType:"MDXLayout"}),(0,a.yg)("h2",{id:"google-cloud-storage-extension"},"Google Cloud Storage Extension"),(0,a.yg)("p",null,"This extension allows you to do 2 things:"),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("a",{parentName:"li",href:"#reading-data-from-google-cloud-storage"},"Ingest data")," from files stored in Google Cloud Storage."),(0,a.yg)("li",{parentName:"ul"},"Write segments to ",(0,a.yg)("a",{parentName:"li",href:"#deep-storage"},"deep storage")," in GCS.")),(0,a.yg)("p",null,"To use this Apache Druid extension, ",(0,a.yg)("a",{parentName:"p",href:"/docs/latest/configuration/extensions#loading-extensions"},"include")," ",(0,a.yg)("inlineCode",{parentName:"p"},"druid-google-extensions")," in the extensions load list."),(0,a.yg)("h3",{id:"required-configuration"},"Required Configuration"),(0,a.yg)("p",null,"To configure connectivity to google cloud, run druid processes with ",(0,a.yg)("inlineCode",{parentName:"p"},"GOOGLE_APPLICATION_CREDENTIALS=/path/to/service_account_keyfile")," in the environment."),(0,a.yg)("h3",{id:"reading-data-from-google-cloud-storage"},"Reading data from Google Cloud Storage"),(0,a.yg)("p",null,"The ",(0,a.yg)("a",{parentName:"p",href:"/docs/latest/ingestion/input-sources"},"Google Cloud Storage input source")," is supported by the ",(0,a.yg)("a",{parentName:"p",href:"/docs/latest/ingestion/native-batch"},"Parallel task"),"\nto read objects directly from Google Cloud Storage. If you use the ",(0,a.yg)("a",{parentName:"p",href:"/docs/latest/ingestion/hadoop"},"Hadoop task"),",\nyou can read data from Google Cloud Storage by specifying the paths in your ",(0,a.yg)("a",{parentName:"p",href:"/docs/latest/ingestion/hadoop#inputspec"},(0,a.yg)("inlineCode",{parentName:"a"},"inputSpec")),"."),(0,a.yg)("h3",{id:"deep-storage"},"Deep Storage"),(0,a.yg)("p",null,"Deep storage can be written to Google Cloud Storage either via this extension or the ",(0,a.yg)("a",{parentName:"p",href:"/docs/latest/development/extensions-core/hdfs"},"druid-hdfs-storage extension"),"."),(0,a.yg)("h4",{id:"configuration"},"Configuration"),(0,a.yg)("p",null,"To configure connectivity to google cloud, run druid processes with ",(0,a.yg)("inlineCode",{parentName:"p"},"GOOGLE_APPLICATION_CREDENTIALS=/path/to/service_account_keyfile")," in the environment."),(0,a.yg)("table",null,(0,a.yg)("thead",{parentName:"table"},(0,a.yg)("tr",{parentName:"thead"},(0,a.yg)("th",{parentName:"tr",align:null},"Property"),(0,a.yg)("th",{parentName:"tr",align:null},"Description"),(0,a.yg)("th",{parentName:"tr",align:null},"Possible Values"),(0,a.yg)("th",{parentName:"tr",align:null},"Default"))),(0,a.yg)("tbody",{parentName:"table"},(0,a.yg)("tr",{parentName:"tbody"},(0,a.yg)("td",{parentName:"tr",align:null},(0,a.yg)("inlineCode",{parentName:"td"},"druid.storage.type")),(0,a.yg)("td",{parentName:"tr",align:null},"google"),(0,a.yg)("td",{parentName:"tr",align:null}),(0,a.yg)("td",{parentName:"tr",align:null},"Must be set.")),(0,a.yg)("tr",{parentName:"tbody"},(0,a.yg)("td",{parentName:"tr",align:null},(0,a.yg)("inlineCode",{parentName:"td"},"druid.google.bucket")),(0,a.yg)("td",{parentName:"tr",align:null}),(0,a.yg)("td",{parentName:"tr",align:null},"Google Storage bucket name."),(0,a.yg)("td",{parentName:"tr",align:null},"Must be set.")),(0,a.yg)("tr",{parentName:"tbody"},(0,a.yg)("td",{parentName:"tr",align:null},(0,a.yg)("inlineCode",{parentName:"td"},"druid.google.prefix")),(0,a.yg)("td",{parentName:"tr",align:null},"A prefix string that will be prepended to the blob names for the segments published to Google deep storage"),(0,a.yg)("td",{parentName:"tr",align:null}),(0,a.yg)("td",{parentName:"tr",align:null},'""')),(0,a.yg)("tr",{parentName:"tbody"},(0,a.yg)("td",{parentName:"tr",align:null},(0,a.yg)("inlineCode",{parentName:"td"},"druid.google.maxListingLength")),(0,a.yg)("td",{parentName:"tr",align:null},"maximum number of input files matching a given prefix to retrieve at a time"),(0,a.yg)("td",{parentName:"tr",align:null}),(0,a.yg)("td",{parentName:"tr",align:null},"1024")))))}y.isMDXComponent=!0}}]); |