| "use strict";(self.webpackChunk=self.webpackChunk||[]).push([[404],{15680:(e,r,o)=>{o.d(r,{xA:()=>l,yg:()=>g});var t=o(96540);function n(e,r,o){return r in e?Object.defineProperty(e,r,{value:o,enumerable:!0,configurable:!0,writable:!0}):e[r]=o,e}function i(e,r){var o=Object.keys(e);if(Object.getOwnPropertySymbols){var t=Object.getOwnPropertySymbols(e);r&&(t=t.filter((function(r){return Object.getOwnPropertyDescriptor(e,r).enumerable}))),o.push.apply(o,t)}return o}function a(e){for(var r=1;r<arguments.length;r++){var o=null!=arguments[r]?arguments[r]:{};r%2?i(Object(o),!0).forEach((function(r){n(e,r,o[r])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(o)):i(Object(o)).forEach((function(r){Object.defineProperty(e,r,Object.getOwnPropertyDescriptor(o,r))}))}return e}function s(e,r){if(null==e)return{};var o,t,n=function(e,r){if(null==e)return{};var o,t,n={},i=Object.keys(e);for(t=0;t<i.length;t++)o=i[t],r.indexOf(o)>=0||(n[o]=e[o]);return n}(e,r);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(t=0;t<i.length;t++)o=i[t],r.indexOf(o)>=0||Object.prototype.propertyIsEnumerable.call(e,o)&&(n[o]=e[o])}return n}var d=t.createContext({}),p=function(e){var r=t.useContext(d),o=r;return e&&(o="function"==typeof e?e(r):a(a({},r),e)),o},l=function(e){var r=p(e.components);return t.createElement(d.Provider,{value:r},e.children)},u="mdxType",c={inlineCode:"code",wrapper:function(e){var r=e.children;return t.createElement(t.Fragment,{},r)}},m=t.forwardRef((function(e,r){var o=e.components,n=e.mdxType,i=e.originalType,d=e.parentName,l=s(e,["components","mdxType","originalType","parentName"]),u=p(o),m=n,g=u["".concat(d,".").concat(m)]||u[m]||c[m]||i;return o?t.createElement(g,a(a({ref:r},l),{},{components:o})):t.createElement(g,a({ref:r},l))}));function g(e,r){var o=arguments,n=r&&r.mdxType;if("string"==typeof e||n){var i=o.length,a=new Array(i);a[0]=m;var s={};for(var d in r)hasOwnProperty.call(r,d)&&(s[d]=r[d]);s.originalType=e,s[u]="string"==typeof e?e:n,a[1]=s;for(var p=2;p<i;p++)a[p]=o[p];return t.createElement.apply(null,a)}return t.createElement.apply(null,o)}m.displayName="MDXCreateElement"},53414:(e,r,o)=>{o.r(r),o.d(r,{assets:()=>l,contentTitle:()=>d,default:()=>g,frontMatter:()=>s,metadata:()=>p,toc:()=>u});var t=o(58168),n=o(98587),i=(o(96540),o(15680)),a=["components"],s={id:"tutorial-kerberos-hadoop",title:"Configure Apache Druid to use Kerberized Apache Hadoop as deep storage",sidebar_label:"Kerberized HDFS deep storage"},d=void 0,p={unversionedId:"tutorials/tutorial-kerberos-hadoop",id:"tutorials/tutorial-kerberos-hadoop",title:"Configure Apache Druid to use Kerberized Apache Hadoop as deep storage",description:"\x3c!--",source:"@site/docs/29.0.0/tutorials/tutorial-kerberos-hadoop.md",sourceDirName:"tutorials",slug:"/tutorials/tutorial-kerberos-hadoop",permalink:"/docs/29.0.0/tutorials/tutorial-kerberos-hadoop",draft:!1,tags:[],version:"current",frontMatter:{id:"tutorial-kerberos-hadoop",title:"Configure Apache Druid to use Kerberized Apache Hadoop as deep storage",sidebar_label:"Kerberized HDFS deep storage"},sidebar:"docs",previous:{title:"Run with Docker",permalink:"/docs/29.0.0/tutorials/docker"},next:{title:"Get to know Query view",permalink:"/docs/29.0.0/tutorials/tutorial-sql-query-view"}},l={},u=[{value:"Hadoop Setup",id:"hadoop-setup",level:2},{value:"HDFS Folders and permissions",id:"hdfs-folders-and-permissions",level:3},{value:"Druid Setup",id:"druid-setup",level:2},{value:"common.runtime.properties",id:"commonruntimeproperties",level:3},{value:"Hadoop Jars",id:"hadoop-jars",level:3},{value:"Kerberos setup",id:"kerberos-setup",level:3},{value:"Restart Druid Services",id:"restart-druid-services",level:3}],c={toc:u},m="wrapper";function g(e){var r=e.components,o=(0,n.A)(e,a);return(0,i.yg)(m,(0,t.A)({},c,o,{components:r,mdxType:"MDXLayout"}),(0,i.yg)("h2",{id:"hadoop-setup"},"Hadoop Setup"),(0,i.yg)("p",null,"Following are the configurations files required to be copied over to Druid conf folders:"),(0,i.yg)("ol",null,(0,i.yg)("li",{parentName:"ol"},"For HDFS as a deep storage, hdfs-site.xml, core-site.xml"),(0,i.yg)("li",{parentName:"ol"},"For ingestion, mapred-site.xml, yarn-site.xml")),(0,i.yg)("h3",{id:"hdfs-folders-and-permissions"},"HDFS Folders and permissions"),(0,i.yg)("ol",null,(0,i.yg)("li",{parentName:"ol"},(0,i.yg)("p",{parentName:"li"},"Choose any folder name for the druid deep storage, for example 'druid'")),(0,i.yg)("li",{parentName:"ol"},(0,i.yg)("p",{parentName:"li"},"Create the folder in hdfs under the required parent folder. For example,\n",(0,i.yg)("inlineCode",{parentName:"p"},"hdfs dfs -mkdir /druid"),"\nOR\n",(0,i.yg)("inlineCode",{parentName:"p"},"hdfs dfs -mkdir /apps/druid"))),(0,i.yg)("li",{parentName:"ol"},(0,i.yg)("p",{parentName:"li"},"Give druid processes appropriate permissions for the druid processes to access this folder. This would ensure that druid is able to create necessary folders like data and indexing_log in HDFS.\nFor example, if druid processes run as user 'root', then"),(0,i.yg)("pre",{parentName:"li"},(0,i.yg)("code",{parentName:"pre"},"`hdfs dfs -chown root:root /apps/druid`\n\nOR\n\n`hdfs dfs -chmod 777 /apps/druid`\n")))),(0,i.yg)("p",null,"Druid creates necessary sub-folders to store data and index under this newly created folder."),(0,i.yg)("h2",{id:"druid-setup"},"Druid Setup"),(0,i.yg)("p",null,"Edit common.runtime.properties at conf/druid/_common/common.runtime.properties to include the HDFS properties. Folders used for the location are same as the ones used for example above."),(0,i.yg)("h3",{id:"commonruntimeproperties"},"common.runtime.properties"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-properties"},"# Deep storage\n#\n# For HDFS:\ndruid.storage.type=hdfs\ndruid.storage.storageDirectory=/druid/segments\n# OR\n# druid.storage.storageDirectory=/apps/druid/segments\n\n#\n# Indexing service logs\n#\n\n# For HDFS:\ndruid.indexer.logs.type=hdfs\ndruid.indexer.logs.directory=/druid/indexing-logs\n# OR\n# druid.storage.storageDirectory=/apps/druid/indexing-logs\n")),(0,i.yg)("p",null,"Note: Comment out Local storage and S3 Storage parameters in the file"),(0,i.yg)("p",null,"Also include hdfs-storage core extension to ",(0,i.yg)("inlineCode",{parentName:"p"},"conf/druid/_common/common.runtime.properties")),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-properties"},'#\n# Extensions\n#\n\ndruid.extensions.directory=dist/druid/extensions\ndruid.extensions.hadoopDependenciesDir=dist/druid/hadoop-dependencies\ndruid.extensions.loadList=["mysql-metadata-storage", "druid-hdfs-storage", "druid-kerberos"]\n')),(0,i.yg)("h3",{id:"hadoop-jars"},"Hadoop Jars"),(0,i.yg)("p",null,"Ensure that Druid has necessary jars to support the Hadoop version."),(0,i.yg)("p",null,"Find the hadoop version using command, ",(0,i.yg)("inlineCode",{parentName:"p"},"hadoop version")),(0,i.yg)("p",null,"In case there is other software used with hadoop, like ",(0,i.yg)("inlineCode",{parentName:"p"},"WanDisco"),", ensure that"),(0,i.yg)("ol",null,(0,i.yg)("li",{parentName:"ol"},"the necessary libraries are available"),(0,i.yg)("li",{parentName:"ol"},"add the requisite extensions to ",(0,i.yg)("inlineCode",{parentName:"li"},"druid.extensions.loadlist")," in ",(0,i.yg)("inlineCode",{parentName:"li"},"conf/druid/_common/common.runtime.properties"))),(0,i.yg)("h3",{id:"kerberos-setup"},"Kerberos setup"),(0,i.yg)("p",null,"Create a headless keytab which would have access to the druid data and index."),(0,i.yg)("p",null,"Edit conf/druid/_common/common.runtime.properties and add the following properties:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-properties"},"druid.hadoop.security.kerberos.principal\ndruid.hadoop.security.kerberos.keytab\n")),(0,i.yg)("p",null,"For example"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-properties"},"druid.hadoop.security.kerberos.principal=hdfs-test@EXAMPLE.IO\ndruid.hadoop.security.kerberos.keytab=/etc/security/keytabs/hdfs.headless.keytab\n")),(0,i.yg)("h3",{id:"restart-druid-services"},"Restart Druid Services"),(0,i.yg)("p",null,"With the above changes, restart Druid. This would ensure that Druid works with Kerberized Hadoop"))}g.isMDXComponent=!0}}]); |