"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[5940],{15680:(e,t,n)=>{n.d(t,{xA:()=>d,yg:()=>c});var a=n(96540);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function i(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function o(e,t){if(null==e)return{};var n,a,r=function(e,t){if(null==e)return{};var n,a,r={},l=Object.keys(e);for(a=0;a<l.length;a++)n=l[a],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(a=0;a<l.length;a++)n=l[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var g=a.createContext({}),p=function(e){var t=a.useContext(g),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},d=function(e){var t=p(e.components);return a.createElement(g.Provider,{value:t},e.children)},s="mdxType",u={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},y=a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,l=e.originalType,g=e.parentName,d=o(e,["components","mdxType","originalType","parentName"]),s=p(n),y=r,c=s["".concat(g,".").concat(y)]||s[y]||u[y]||l;return n?a.createElement(c,i(i({ref:t},d),{},{components:n})):a.createElement(c,i({ref:t},d))}));function c(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var l=n.length,i=new Array(l);i[0]=y;var o={};for(var g in t)hasOwnProperty.call(t,g)&&(o[g]=t[g]);o.originalType=e,o[s]="string"==typeof e?e:r,i[1]=o;for(var p=2;p<l;p++)i[p]=n[p];return a.createElement.apply(null,i)}return a.createElement.apply(null,n)}y.displayName="MDXCreateElement"},90255:(e,t,n)=>{n.r(t),n.d(t,{assets:()=>d,contentTitle:()=>g,default:()=>c,frontMatter:()=>o,metadata:()=>p,toc:()=>s});var a=n(58168),r=n(98587),l=(n(96540),n(15680)),i=["components"],o={id:"cloudfiles",title:"Rackspace Cloud Files"},g=void 0,p={unversionedId:"development/extensions-contrib/cloudfiles",id:"development/extensions-contrib/cloudfiles",title:"Rackspace Cloud Files",description:"\x3c!--",source:"@site/docs/29.0.0/development/extensions-contrib/cloudfiles.md",sourceDirName:"development/extensions-contrib",slug:"/development/extensions-contrib/cloudfiles",permalink:"/docs/29.0.0/development/extensions-contrib/cloudfiles",draft:!1,tags:[],version:"current",frontMatter:{id:"cloudfiles",title:"Rackspace Cloud Files"}},d={},s=[{value:"Deep Storage",id:"deep-storage",level:2},{value:"Firehose",id:"firehose",level:2},{value:"StaticCloudFilesFirehose",id:"staticcloudfilesfirehose",level:4}],u={toc:s},y="wrapper";function c(e){var t=e.components,n=(0,r.A)(e,i);return(0,l.yg)(y,(0,a.A)({},u,n,{components:t,mdxType:"MDXLayout"}),(0,l.yg)("p",null,"To use this Apache Druid extension, ",(0,l.yg)("a",{parentName:"p",href:"/docs/29.0.0/configuration/extensions#loading-extensions"},"include")," ",(0,l.yg)("inlineCode",{parentName:"p"},"druid-cloudfiles-extensions")," in the extensions load list."),(0,l.yg)("h2",{id:"deep-storage"},"Deep Storage"),(0,l.yg)("p",null,(0,l.yg)("a",{parentName:"p",href:"http://www.rackspace.com/cloud/files/"},"Rackspace Cloud Files")," is another option for deep storage. This requires some additional Druid configuration."),(0,l.yg)("table",null,(0,l.yg)("thead",{parentName:"table"},(0,l.yg)("tr",{parentName:"thead"},(0,l.yg)("th",{parentName:"tr",align:null},"Property"),(0,l.yg)("th",{parentName:"tr",align:null},"Possible Values"),(0,l.yg)("th",{parentName:"tr",align:null},"Description"),(0,l.yg)("th",{parentName:"tr",align:null},"Default"))),(0,l.yg)("tbody",{parentName:"table"},(0,l.yg)("tr",{parentName:"tbody"},(0,l.yg)("td",{parentName:"tr",align:null},(0,l.yg)("inlineCode",{parentName:"td"},"druid.storage.type")),(0,l.yg)("td",{parentName:"tr",align:null},"cloudfiles"),(0,l.yg)("td",{parentName:"tr",align:null}),(0,l.yg)("td",{parentName:"tr",align:null},"Must be set.")),(0,l.yg)("tr",{parentName:"tbody"},(0,l.yg)("td",{parentName:"tr",align:null},(0,l.yg)("inlineCode",{parentName:"td"},"druid.storage.region")),(0,l.yg)("td",{parentName:"tr",align:null}),(0,l.yg)("td",{parentName:"tr",align:null},"Rackspace Cloud Files region."),(0,l.yg)("td",{parentName:"tr",align:null},"Must be set.")),(0,l.yg)("tr",{parentName:"tbody"},(0,l.yg)("td",{parentName:"tr",align:null},(0,l.yg)("inlineCode",{parentName:"td"},"druid.storage.container")),(0,l.yg)("td",{parentName:"tr",align:null}),(0,l.yg)("td",{parentName:"tr",align:null},"Rackspace Cloud Files container name."),(0,l.yg)("td",{parentName:"tr",align:null},"Must be set.")),(0,l.yg)("tr",{parentName:"tbody"},(0,l.yg)("td",{parentName:"tr",align:null},(0,l.yg)("inlineCode",{parentName:"td"},"druid.storage.basePath")),(0,l.yg)("td",{parentName:"tr",align:null}),(0,l.yg)("td",{parentName:"tr",align:null},"Rackspace Cloud Files base path to use in the container."),(0,l.yg)("td",{parentName:"tr",align:null},"Must be set.")),(0,l.yg)("tr",{parentName:"tbody"},(0,l.yg)("td",{parentName:"tr",align:null},(0,l.yg)("inlineCode",{parentName:"td"},"druid.storage.operationMaxRetries")),(0,l.yg)("td",{parentName:"tr",align:null}),(0,l.yg)("td",{parentName:"tr",align:null},"Number of tries before cancel a Rackspace operation."),(0,l.yg)("td",{parentName:"tr",align:null},"10")),(0,l.yg)("tr",{parentName:"tbody"},(0,l.yg)("td",{parentName:"tr",align:null},(0,l.yg)("inlineCode",{parentName:"td"},"druid.cloudfiles.userName")),(0,l.yg)("td",{parentName:"tr",align:null}),(0,l.yg)("td",{parentName:"tr",align:null},"Rackspace Cloud username"),(0,l.yg)("td",{parentName:"tr",align:null},"Must be set.")),(0,l.yg)("tr",{parentName:"tbody"},(0,l.yg)("td",{parentName:"tr",align:null},(0,l.yg)("inlineCode",{parentName:"td"},"druid.cloudfiles.apiKey")),(0,l.yg)("td",{parentName:"tr",align:null}),(0,l.yg)("td",{parentName:"tr",align:null},"Rackspace Cloud API key."),(0,l.yg)("td",{parentName:"tr",align:null},"Must be set.")),(0,l.yg)("tr",{parentName:"tbody"},(0,l.yg)("td",{parentName:"tr",align:null},(0,l.yg)("inlineCode",{parentName:"td"},"druid.cloudfiles.provider")),(0,l.yg)("td",{parentName:"tr",align:null},"rackspace-cloudfiles-us,rackspace-cloudfiles-uk"),(0,l.yg)("td",{parentName:"tr",align:null},"Name of the provider depending on the region."),(0,l.yg)("td",{parentName:"tr",align:null},"Must be set.")),(0,l.yg)("tr",{parentName:"tbody"},(0,l.yg)("td",{parentName:"tr",align:null},(0,l.yg)("inlineCode",{parentName:"td"},"druid.cloudfiles.useServiceNet")),(0,l.yg)("td",{parentName:"tr",align:null},"true,false"),(0,l.yg)("td",{parentName:"tr",align:null},"Whether to use the internal service net."),(0,l.yg)("td",{parentName:"tr",align:null},"true")))),(0,l.yg)("h2",{id:"firehose"},"Firehose"),(0,l.yg)("a",{name:"firehose"}),(0,l.yg)("h4",{id:"staticcloudfilesfirehose"},"StaticCloudFilesFirehose"),(0,l.yg)("p",null,"This firehose ingests events, similar to the StaticAzureBlobStoreFirehose, but from Rackspace's Cloud Files."),(0,l.yg)("p",null,"Data is newline delimited, with one JSON object per line and parsed as per the ",(0,l.yg)("inlineCode",{parentName:"p"},"InputRowParser")," configuration."),(0,l.yg)("p",null,"The storage account is shared with the one used for Rackspace's Cloud Files deep storage functionality, but blobs can be in a different region and container."),(0,l.yg)("p",null,"As with the Azure blobstore, it is assumed to be gzipped if the extension ends in .gz"),(0,l.yg)("p",null,"This firehose is ",(0,l.yg)("em",{parentName:"p"},"splittable")," and can be used by ",(0,l.yg)("a",{parentName:"p",href:"/docs/29.0.0/ingestion/native-batch"},"native parallel index tasks"),".\nSince each split represents an object in this firehose, each worker task of ",(0,l.yg)("inlineCode",{parentName:"p"},"index_parallel")," will read an object."),(0,l.yg)("p",null,"Sample spec:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-json"},'"firehose" : {\n    "type" : "static-cloudfiles",\n    "blobs": [\n        {\n          "region": "DFW"\n          "container": "container",\n          "path": "/path/to/your/file.json"\n        },\n        {\n          "region": "ORD"\n          "container": "anothercontainer",\n          "path": "/another/path.json"\n        }\n    ]\n}\n')),(0,l.yg)("p",null,"This firehose provides caching and prefetching features. In IndexTask, a firehose can be read twice if intervals or\nshardSpecs are not specified, and, in this case, caching can be useful. Prefetching is preferred when direct scan of objects is slow."),(0,l.yg)("table",null,(0,l.yg)("thead",{parentName:"table"},(0,l.yg)("tr",{parentName:"thead"},(0,l.yg)("th",{parentName:"tr",align:null},"property"),(0,l.yg)("th",{parentName:"tr",align:null},"description"),(0,l.yg)("th",{parentName:"tr",align:null},"default"),(0,l.yg)("th",{parentName:"tr",align:null},"required?"))),(0,l.yg)("tbody",{parentName:"table"},(0,l.yg)("tr",{parentName:"tbody"},(0,l.yg)("td",{parentName:"tr",align:null},"type"),(0,l.yg)("td",{parentName:"tr",align:null},"This should be ",(0,l.yg)("inlineCode",{parentName:"td"},"static-cloudfiles"),"."),(0,l.yg)("td",{parentName:"tr",align:null},"N/A"),(0,l.yg)("td",{parentName:"tr",align:null},"yes")),(0,l.yg)("tr",{parentName:"tbody"},(0,l.yg)("td",{parentName:"tr",align:null},"blobs"),(0,l.yg)("td",{parentName:"tr",align:null},"JSON array of Cloud Files blobs."),(0,l.yg)("td",{parentName:"tr",align:null},"N/A"),(0,l.yg)("td",{parentName:"tr",align:null},"yes")),(0,l.yg)("tr",{parentName:"tbody"},(0,l.yg)("td",{parentName:"tr",align:null},"maxCacheCapacityBytes"),(0,l.yg)("td",{parentName:"tr",align:null},"Maximum size of the cache space in bytes. 0 means disabling cache."),(0,l.yg)("td",{parentName:"tr",align:null},"1073741824"),(0,l.yg)("td",{parentName:"tr",align:null},"no")),(0,l.yg)("tr",{parentName:"tbody"},(0,l.yg)("td",{parentName:"tr",align:null},"maxCacheCapacityBytes"),(0,l.yg)("td",{parentName:"tr",align:null},"Maximum size of the cache space in bytes. 0 means disabling cache. Cached files are not removed until the ingestion task completes."),(0,l.yg)("td",{parentName:"tr",align:null},"1073741824"),(0,l.yg)("td",{parentName:"tr",align:null},"no")),(0,l.yg)("tr",{parentName:"tbody"},(0,l.yg)("td",{parentName:"tr",align:null},"maxFetchCapacityBytes"),(0,l.yg)("td",{parentName:"tr",align:null},"Maximum size of the fetch space in bytes. 0 means disabling prefetch. Prefetched files are removed immediately once they are read."),(0,l.yg)("td",{parentName:"tr",align:null},"1073741824"),(0,l.yg)("td",{parentName:"tr",align:null},"no")),(0,l.yg)("tr",{parentName:"tbody"},(0,l.yg)("td",{parentName:"tr",align:null},"fetchTimeout"),(0,l.yg)("td",{parentName:"tr",align:null},"Timeout for fetching a Cloud Files object."),(0,l.yg)("td",{parentName:"tr",align:null},"60000"),(0,l.yg)("td",{parentName:"tr",align:null},"no")),(0,l.yg)("tr",{parentName:"tbody"},(0,l.yg)("td",{parentName:"tr",align:null},"maxFetchRetry"),(0,l.yg)("td",{parentName:"tr",align:null},"Maximum retry for fetching a Cloud Files object."),(0,l.yg)("td",{parentName:"tr",align:null},"3"),(0,l.yg)("td",{parentName:"tr",align:null},"no")))),(0,l.yg)("p",null,"Cloud Files Blobs:"),(0,l.yg)("table",null,(0,l.yg)("thead",{parentName:"table"},(0,l.yg)("tr",{parentName:"thead"},(0,l.yg)("th",{parentName:"tr",align:null},"property"),(0,l.yg)("th",{parentName:"tr",align:null},"description"),(0,l.yg)("th",{parentName:"tr",align:null},"default"),(0,l.yg)("th",{parentName:"tr",align:null},"required?"))),(0,l.yg)("tbody",{parentName:"table"},(0,l.yg)("tr",{parentName:"tbody"},(0,l.yg)("td",{parentName:"tr",align:null},"container"),(0,l.yg)("td",{parentName:"tr",align:null},"Name of the Cloud Files container"),(0,l.yg)("td",{parentName:"tr",align:null},"N/A"),(0,l.yg)("td",{parentName:"tr",align:null},"yes")),(0,l.yg)("tr",{parentName:"tbody"},(0,l.yg)("td",{parentName:"tr",align:null},"path"),(0,l.yg)("td",{parentName:"tr",align:null},"The path where data is located."),(0,l.yg)("td",{parentName:"tr",align:null},"N/A"),(0,l.yg)("td",{parentName:"tr",align:null},"yes")))))}c.isMDXComponent=!0}}]);