| "use strict";(self.webpackChunk=self.webpackChunk||[]).push([[3804],{15680:(e,n,o)=>{o.d(n,{xA:()=>s,yg:()=>m});var t=o(96540);function a(e,n,o){return n in e?Object.defineProperty(e,n,{value:o,enumerable:!0,configurable:!0,writable:!0}):e[n]=o,e}function r(e,n){var o=Object.keys(e);if(Object.getOwnPropertySymbols){var t=Object.getOwnPropertySymbols(e);n&&(t=t.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),o.push.apply(o,t)}return o}function p(e){for(var n=1;n<arguments.length;n++){var o=null!=arguments[n]?arguments[n]:{};n%2?r(Object(o),!0).forEach((function(n){a(e,n,o[n])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(o)):r(Object(o)).forEach((function(n){Object.defineProperty(e,n,Object.getOwnPropertyDescriptor(o,n))}))}return e}function i(e,n){if(null==e)return{};var o,t,a=function(e,n){if(null==e)return{};var o,t,a={},r=Object.keys(e);for(t=0;t<r.length;t++)o=r[t],n.indexOf(o)>=0||(a[o]=e[o]);return a}(e,n);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(t=0;t<r.length;t++)o=r[t],n.indexOf(o)>=0||Object.prototype.propertyIsEnumerable.call(e,o)&&(a[o]=e[o])}return a}var l=t.createContext({}),d=function(e){var n=t.useContext(l),o=n;return e&&(o="function"==typeof e?e(n):p(p({},n),e)),o},s=function(e){var n=d(e.components);return t.createElement(l.Provider,{value:n},e.children)},y="mdxType",u={inlineCode:"code",wrapper:function(e){var n=e.children;return t.createElement(t.Fragment,{},n)}},c=t.forwardRef((function(e,n){var o=e.components,a=e.mdxType,r=e.originalType,l=e.parentName,s=i(e,["components","mdxType","originalType","parentName"]),y=d(o),c=a,m=y["".concat(l,".").concat(c)]||y[c]||u[c]||r;return o?t.createElement(m,p(p({ref:n},s),{},{components:o})):t.createElement(m,p({ref:n},s))}));function m(e,n){var o=arguments,a=n&&n.mdxType;if("string"==typeof e||a){var r=o.length,p=new Array(r);p[0]=c;var i={};for(var l in n)hasOwnProperty.call(n,l)&&(i[l]=n[l]);i.originalType=e,i[y]="string"==typeof e?e:a,p[1]=i;for(var d=2;d<r;d++)p[d]=o[d];return t.createElement.apply(null,p)}return t.createElement.apply(null,o)}c.displayName="MDXCreateElement"},75479:(e,n,o)=>{o.r(n),o.d(n,{assets:()=>s,contentTitle:()=>l,default:()=>m,frontMatter:()=>i,metadata:()=>d,toc:()=>y});var t=o(58168),a=o(98587),r=(o(96540),o(15680)),p=["components"],i={id:"pull-deps",title:"pull-deps tool"},l=void 0,d={unversionedId:"operations/pull-deps",id:"operations/pull-deps",title:"pull-deps tool",description:"\x3c!--",source:"@site/docs/latest/operations/pull-deps.md",sourceDirName:"operations",slug:"/operations/pull-deps",permalink:"/docs/latest/operations/pull-deps",draft:!1,tags:[],version:"current",frontMatter:{id:"pull-deps",title:"pull-deps tool"},sidebar:"docs",previous:{title:"insert-segment-to-db tool",permalink:"/docs/latest/operations/insert-segment-to-db"},next:{title:"Deep storage migration",permalink:"/docs/latest/operations/deep-storage-migration"}},s={},y=[],u={toc:y},c="wrapper";function m(e){var n=e.components,o=(0,a.A)(e,p);return(0,r.yg)(c,(0,t.A)({},u,o,{components:n,mdxType:"MDXLayout"}),(0,r.yg)("p",null,(0,r.yg)("inlineCode",{parentName:"p"},"pull-deps")," is an Apache Druid tool that can pull down dependencies to the local repository and lay dependencies out into the extension directory as needed."),(0,r.yg)("p",null,(0,r.yg)("inlineCode",{parentName:"p"},"pull-deps")," has several command line options, they are as follows:"),(0,r.yg)("p",null,(0,r.yg)("inlineCode",{parentName:"p"},"-c")," or ",(0,r.yg)("inlineCode",{parentName:"p"},"--coordinate")," (Can be specified multiple times)"),(0,r.yg)("p",null,"Extension coordinate to pull down, followed by a maven coordinate, e.g. org.apache.druid.extensions:mysql-metadata-storage"),(0,r.yg)("p",null,(0,r.yg)("inlineCode",{parentName:"p"},"-h")," or ",(0,r.yg)("inlineCode",{parentName:"p"},"--hadoop-coordinate")," (Can be specified multiply times)"),(0,r.yg)("p",null,"Apache Hadoop dependency to pull down, followed by a maven coordinate, e.g. org.apache.hadoop:hadoop-client:2.4.0"),(0,r.yg)("p",null,(0,r.yg)("inlineCode",{parentName:"p"},"--no-default-hadoop")),(0,r.yg)("p",null,"Don't pull down the default hadoop coordinate, i.e., org.apache.hadoop:hadoop-client:2.3.0. If ",(0,r.yg)("inlineCode",{parentName:"p"},"-h")," option is supplied, then default hadoop coordinate will not be downloaded."),(0,r.yg)("p",null,(0,r.yg)("inlineCode",{parentName:"p"},"--clean")),(0,r.yg)("p",null,"Remove existing extension and hadoop dependencies directories before pulling down dependencies."),(0,r.yg)("p",null,(0,r.yg)("inlineCode",{parentName:"p"},"-l")," or ",(0,r.yg)("inlineCode",{parentName:"p"},"--localRepository")),(0,r.yg)("p",null,"A local repository that Maven will use to put downloaded files. Then pull-deps will lay these files out into the extensions directory as needed."),(0,r.yg)("p",null,(0,r.yg)("inlineCode",{parentName:"p"},"-r")," or ",(0,r.yg)("inlineCode",{parentName:"p"},"--remoteRepository")),(0,r.yg)("p",null,"Add a remote repository. Unless ",(0,r.yg)("inlineCode",{parentName:"p"},"--no-default-remote-repositories")," is provided, these will be used after ",(0,r.yg)("a",{parentName:"p",href:"https://repo1.maven.org/maven2/"},"https://repo1.maven.org/maven2/"),"."),(0,r.yg)("p",null,(0,r.yg)("inlineCode",{parentName:"p"},"--no-default-remote-repositories")),(0,r.yg)("p",null,"Don't use the default remote repository, ",(0,r.yg)("a",{parentName:"p",href:"https://repo1.maven.org/maven2/"},"https://repo1.maven.org/maven2/"),". Only use the repositories provided directly via --remoteRepository."),(0,r.yg)("p",null,(0,r.yg)("inlineCode",{parentName:"p"},"-d")," or ",(0,r.yg)("inlineCode",{parentName:"p"},"--defaultVersion")),(0,r.yg)("p",null,"Version to use for extension coordinate that doesn't have a version information. For example, if extension coordinate is ",(0,r.yg)("inlineCode",{parentName:"p"},"org.apache.druid.extensions:mysql-metadata-storage"),", and default version is ",(0,r.yg)("inlineCode",{parentName:"p"},"29.0.0"),", then this coordinate will be treated as ",(0,r.yg)("inlineCode",{parentName:"p"},"org.apache.druid.extensions:mysql-metadata-storage:29.0.0")),(0,r.yg)("p",null,(0,r.yg)("inlineCode",{parentName:"p"},"--use-proxy")),(0,r.yg)("p",null,"Use http/https proxy to send request to the remote repository servers. ",(0,r.yg)("inlineCode",{parentName:"p"},"--proxy-host")," and ",(0,r.yg)("inlineCode",{parentName:"p"},"--proxy-port")," must be set explicitly if this option is enabled."),(0,r.yg)("p",null,(0,r.yg)("inlineCode",{parentName:"p"},"--proxy-type")),(0,r.yg)("p",null,"Set the proxy type, Should be either ",(0,r.yg)("em",{parentName:"p"},"http")," or ",(0,r.yg)("em",{parentName:"p"},"https"),", default value is ",(0,r.yg)("em",{parentName:"p"},"https"),"."),(0,r.yg)("p",null,(0,r.yg)("inlineCode",{parentName:"p"},"--proxy-host")),(0,r.yg)("p",null,"Set the proxy host. e.g. proxy.com."),(0,r.yg)("p",null,(0,r.yg)("inlineCode",{parentName:"p"},"--proxy-port")),(0,r.yg)("p",null,"Set the proxy port number. e.g. 8080."),(0,r.yg)("p",null,(0,r.yg)("inlineCode",{parentName:"p"},"--proxy-username")),(0,r.yg)("p",null,"Set a username to connect to the proxy, this option is only required if the proxy server uses authentication."),(0,r.yg)("p",null,(0,r.yg)("inlineCode",{parentName:"p"},"--proxy-password")),(0,r.yg)("p",null,"Set a password to connect to the proxy, this option is only required if the proxy server uses authentication."),(0,r.yg)("p",null,"To run ",(0,r.yg)("inlineCode",{parentName:"p"},"pull-deps"),", you should"),(0,r.yg)("p",null,"1) Specify ",(0,r.yg)("inlineCode",{parentName:"p"},"druid.extensions.directory")," and ",(0,r.yg)("inlineCode",{parentName:"p"},"druid.extensions.hadoopDependenciesDir"),", these two properties tell ",(0,r.yg)("inlineCode",{parentName:"p"},"pull-deps")," where to put extensions. If you don't specify them, default values will be used, see ",(0,r.yg)("a",{parentName:"p",href:"/docs/latest/configuration/"},"Configuration"),"."),(0,r.yg)("p",null,"2) Tell ",(0,r.yg)("inlineCode",{parentName:"p"},"pull-deps")," what to download using ",(0,r.yg)("inlineCode",{parentName:"p"},"-c")," or ",(0,r.yg)("inlineCode",{parentName:"p"},"-h")," option, which are followed by a maven coordinate."),(0,r.yg)("p",null,"Example:"),(0,r.yg)("p",null,"Suppose you want to download ",(0,r.yg)("inlineCode",{parentName:"p"},"mysql-metadata-storage")," and ",(0,r.yg)("inlineCode",{parentName:"p"},"hadoop-client"),"(both 2.3.0 and 2.4.0) with a specific version, you can run ",(0,r.yg)("inlineCode",{parentName:"p"},"pull-deps")," command with ",(0,r.yg)("inlineCode",{parentName:"p"},"-c org.apache.druid.extensions:mysql-metadata-storage:29.0.0"),", ",(0,r.yg)("inlineCode",{parentName:"p"},"-h org.apache.hadoop:hadoop-client:2.3.0")," and ",(0,r.yg)("inlineCode",{parentName:"p"},"-h org.apache.hadoop:hadoop-client:2.4.0"),", an example command would be:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},'java -classpath "/my/druid/lib/*" org.apache.druid.cli.Main tools pull-deps --clean -c org.apache.druid.extensions:mysql-metadata-storage:29.0.0 -h org.apache.hadoop:hadoop-client:2.3.0 -h org.apache.hadoop:hadoop-client:2.4.0\n')),(0,r.yg)("p",null,"Because ",(0,r.yg)("inlineCode",{parentName:"p"},"--clean")," is supplied, this command will first remove the directories specified at ",(0,r.yg)("inlineCode",{parentName:"p"},"druid.extensions.directory")," and ",(0,r.yg)("inlineCode",{parentName:"p"},"druid.extensions.hadoopDependenciesDir"),", then recreate them and start downloading the extensions there. After finishing downloading, if you go to the extension directories you specified, you will see"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},"tree extensions\nextensions\n\u2514\u2500\u2500 mysql-metadata-storage\n \u2514\u2500\u2500 mysql-metadata-storage-29.0.0.jar\n")),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},"tree hadoop-dependencies\nhadoop-dependencies/\n\u2514\u2500\u2500 hadoop-client\n \u251c\u2500\u2500 2.3.0\n \u2502\xa0\xa0 \u251c\u2500\u2500 activation-1.1.jar\n \u2502\xa0\xa0 \u251c\u2500\u2500 avro-1.7.4.jar\n \u2502\xa0\xa0 \u251c\u2500\u2500 commons-beanutils-1.7.0.jar\n \u2502\xa0\xa0 \u251c\u2500\u2500 commons-beanutils-core-1.8.0.jar\n \u2502\xa0\xa0 \u251c\u2500\u2500 commons-cli-1.2.jar\n \u2502\xa0\xa0 \u251c\u2500\u2500 commons-codec-1.4.jar\n ..... lots of jars\n \u2514\u2500\u2500 2.4.0\n \u251c\u2500\u2500 activation-1.1.jar\n \u251c\u2500\u2500 avro-1.7.4.jar\n \u251c\u2500\u2500 commons-beanutils-1.7.0.jar\n \u251c\u2500\u2500 commons-beanutils-core-1.8.0.jar\n \u251c\u2500\u2500 commons-cli-1.2.jar\n \u251c\u2500\u2500 commons-codec-1.4.jar\n ..... lots of jars\n")),(0,r.yg)("p",null,"Note that if you specify ",(0,r.yg)("inlineCode",{parentName:"p"},"--defaultVersion"),", you don't have to put version information in the coordinate. For example, if you want ",(0,r.yg)("inlineCode",{parentName:"p"},"mysql-metadata-storage")," to use version ",(0,r.yg)("inlineCode",{parentName:"p"},"29.0.0"),", you can change the command above to"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},'java -classpath "/my/druid/lib/*" org.apache.druid.cli.Main tools pull-deps --defaultVersion 29.0.0 --clean -c org.apache.druid.extensions:mysql-metadata-storage -h org.apache.hadoop:hadoop-client:2.3.0 -h org.apache.hadoop:hadoop-client:2.4.0\n')),(0,r.yg)("admonition",{type:"info"},(0,r.yg)("p",{parentName:"admonition"}," Please note to use the pull-deps tool you must know the Maven groupId, artifactId, and version of your extension."),(0,r.yg)("p",{parentName:"admonition"}," For Druid community extensions listed ",(0,r.yg)("a",{parentName:"p",href:"/docs/latest/configuration/extensions"},"here"),', the groupId is "org.apache.druid.extensions.contrib" and the artifactId is the name of the extension.')))}m.isMDXComponent=!0}}]); |