| "use strict";(self.webpackChunk=self.webpackChunk||[]).push([[3804],{28453:(e,o,n)=>{n.d(o,{R:()=>i,x:()=>r});var s=n(96540);const t={},d=s.createContext(t);function i(e){const o=s.useContext(d);return s.useMemo((function(){return"function"==typeof e?e(o):{...o,...e}}),[o,e])}function r(e){let o;return o=e.disableParentContext?"function"==typeof e.components?e.components(t):e.components||t:i(e.components),s.createElement(d.Provider,{value:o},e.children)}},80862:(e,o,n)=>{n.r(o),n.d(o,{assets:()=>a,contentTitle:()=>r,default:()=>p,frontMatter:()=>i,metadata:()=>s,toc:()=>c});const s=JSON.parse('{"id":"operations/pull-deps","title":"pull-deps tool","description":"\x3c!--","source":"@site/docs/latest/operations/pull-deps.md","sourceDirName":"operations","slug":"/operations/pull-deps","permalink":"/docs/latest/operations/pull-deps","draft":false,"unlisted":false,"tags":[],"version":"current","frontMatter":{"id":"pull-deps","title":"pull-deps tool"},"sidebar":"docs","previous":{"title":"insert-segment-to-db tool","permalink":"/docs/latest/operations/insert-segment-to-db"},"next":{"title":"Deep storage migration","permalink":"/docs/latest/operations/deep-storage-migration"}}');var t=n(74848),d=n(28453);const i={id:"pull-deps",title:"pull-deps tool"},r=void 0,a={},c=[];function l(e){const o={a:"a",admonition:"admonition",code:"code",em:"em",li:"li",ol:"ol",p:"p",pre:"pre",...(0,d.R)(),...e.components};return(0,t.jsxs)(t.Fragment,{children:[(0,t.jsxs)(o.p,{children:[(0,t.jsx)(o.code,{children:"pull-deps"})," is an Apache Druid tool that can pull down dependencies to the local repository and lay dependencies out into the extension directory as needed."]}),"\n",(0,t.jsxs)(o.p,{children:[(0,t.jsx)(o.code,{children:"pull-deps"})," has several command line options, they are as follows:"]}),"\n",(0,t.jsxs)(o.p,{children:[(0,t.jsx)(o.code,{children:"-c"})," or ",(0,t.jsx)(o.code,{children:"--coordinate"})," (Can be specified multiple times)"]}),"\n",(0,t.jsxs)(o.p,{children:["Extension coordinate to pull down, followed by a maven coordinate, e.g. org.apache.druid.extensions",":mysql-metadata-storage"]}),"\n",(0,t.jsxs)(o.p,{children:[(0,t.jsx)(o.code,{children:"-h"})," or ",(0,t.jsx)(o.code,{children:"--hadoop-coordinate"})," (Can be specified multiply times)"]}),"\n",(0,t.jsx)(o.p,{children:"Apache Hadoop dependency to pull down, followed by a maven coordinate, e.g. org.apache.hadoop:hadoop-client:2.4.0"}),"\n",(0,t.jsx)(o.p,{children:(0,t.jsx)(o.code,{children:"--no-default-hadoop"})}),"\n",(0,t.jsxs)(o.p,{children:["Don't pull down the default hadoop coordinate, i.e., org.apache.hadoop:hadoop-client:2.3.0. If ",(0,t.jsx)(o.code,{children:"-h"})," option is supplied, then default hadoop coordinate will not be downloaded."]}),"\n",(0,t.jsx)(o.p,{children:(0,t.jsx)(o.code,{children:"--clean"})}),"\n",(0,t.jsx)(o.p,{children:"Remove existing extension and hadoop dependencies directories before pulling down dependencies."}),"\n",(0,t.jsxs)(o.p,{children:[(0,t.jsx)(o.code,{children:"-l"})," or ",(0,t.jsx)(o.code,{children:"--localRepository"})]}),"\n",(0,t.jsx)(o.p,{children:"A local repository that Maven will use to put downloaded files. Then pull-deps will lay these files out into the extensions directory as needed."}),"\n",(0,t.jsxs)(o.p,{children:[(0,t.jsx)(o.code,{children:"-r"})," or ",(0,t.jsx)(o.code,{children:"--remoteRepository"})]}),"\n",(0,t.jsxs)(o.p,{children:["Add a remote repository. Unless ",(0,t.jsx)(o.code,{children:"--no-default-remote-repositories"})," is provided, these will be used after ",(0,t.jsx)(o.a,{href:"https://repo1.maven.org/maven2/",children:"https://repo1.maven.org/maven2/"}),"."]}),"\n",(0,t.jsx)(o.p,{children:(0,t.jsx)(o.code,{children:"--no-default-remote-repositories"})}),"\n",(0,t.jsxs)(o.p,{children:["Don't use the default remote repository, ",(0,t.jsx)(o.a,{href:"https://repo1.maven.org/maven2/",children:"https://repo1.maven.org/maven2/"}),". Only use the repositories provided directly via --remoteRepository."]}),"\n",(0,t.jsxs)(o.p,{children:[(0,t.jsx)(o.code,{children:"-d"})," or ",(0,t.jsx)(o.code,{children:"--defaultVersion"})]}),"\n",(0,t.jsxs)(o.p,{children:["Version to use for extension coordinate that doesn't have a version information. For example, if extension coordinate is ",(0,t.jsx)(o.code,{children:"org.apache.druid.extensions:mysql-metadata-storage"}),", and default version is ",(0,t.jsx)(o.code,{children:"33.0.0"}),", then this coordinate will be treated as ",(0,t.jsx)(o.code,{children:"org.apache.druid.extensions:mysql-metadata-storage:33.0.0"})]}),"\n",(0,t.jsx)(o.p,{children:(0,t.jsx)(o.code,{children:"--use-proxy"})}),"\n",(0,t.jsxs)(o.p,{children:["Use http/https proxy to send request to the remote repository servers. ",(0,t.jsx)(o.code,{children:"--proxy-host"})," and ",(0,t.jsx)(o.code,{children:"--proxy-port"})," must be set explicitly if this option is enabled."]}),"\n",(0,t.jsx)(o.p,{children:(0,t.jsx)(o.code,{children:"--proxy-type"})}),"\n",(0,t.jsxs)(o.p,{children:["Set the proxy type, Should be either ",(0,t.jsx)(o.em,{children:"http"})," or ",(0,t.jsx)(o.em,{children:"https"}),", default value is ",(0,t.jsx)(o.em,{children:"https"}),"."]}),"\n",(0,t.jsx)(o.p,{children:(0,t.jsx)(o.code,{children:"--proxy-host"})}),"\n",(0,t.jsx)(o.p,{children:"Set the proxy host. e.g. proxy.com."}),"\n",(0,t.jsx)(o.p,{children:(0,t.jsx)(o.code,{children:"--proxy-port"})}),"\n",(0,t.jsx)(o.p,{children:"Set the proxy port number. e.g. 8080."}),"\n",(0,t.jsx)(o.p,{children:(0,t.jsx)(o.code,{children:"--proxy-username"})}),"\n",(0,t.jsx)(o.p,{children:"Set a username to connect to the proxy, this option is only required if the proxy server uses authentication."}),"\n",(0,t.jsx)(o.p,{children:(0,t.jsx)(o.code,{children:"--proxy-password"})}),"\n",(0,t.jsx)(o.p,{children:"Set a password to connect to the proxy, this option is only required if the proxy server uses authentication."}),"\n",(0,t.jsxs)(o.p,{children:["To run ",(0,t.jsx)(o.code,{children:"pull-deps"}),", you should"]}),"\n",(0,t.jsxs)(o.ol,{children:["\n",(0,t.jsxs)(o.li,{children:["\n",(0,t.jsxs)(o.p,{children:["Specify ",(0,t.jsx)(o.code,{children:"druid.extensions.directory"})," and ",(0,t.jsx)(o.code,{children:"druid.extensions.hadoopDependenciesDir"}),", these two properties tell ",(0,t.jsx)(o.code,{children:"pull-deps"})," where to put extensions. If you don't specify them, default values will be used, see ",(0,t.jsx)(o.a,{href:"/docs/latest/configuration/",children:"Configuration"}),"."]}),"\n"]}),"\n",(0,t.jsxs)(o.li,{children:["\n",(0,t.jsxs)(o.p,{children:["Tell ",(0,t.jsx)(o.code,{children:"pull-deps"})," what to download using ",(0,t.jsx)(o.code,{children:"-c"})," or ",(0,t.jsx)(o.code,{children:"-h"})," option, which are followed by a maven coordinate."]}),"\n"]}),"\n"]}),"\n",(0,t.jsx)(o.p,{children:"Example:"}),"\n",(0,t.jsxs)(o.p,{children:["Suppose you want to download ",(0,t.jsx)(o.code,{children:"mysql-metadata-storage"})," and ",(0,t.jsx)(o.code,{children:"hadoop-client"}),"(both 2.3.0 and 2.4.0) with a specific version, you can run ",(0,t.jsx)(o.code,{children:"pull-deps"})," command with ",(0,t.jsx)(o.code,{children:"-c org.apache.druid.extensions:mysql-metadata-storage:33.0.0"}),", ",(0,t.jsx)(o.code,{children:"-h org.apache.hadoop:hadoop-client:2.3.0"})," and ",(0,t.jsx)(o.code,{children:"-h org.apache.hadoop:hadoop-client:2.4.0"}),", an example command would be:"]}),"\n",(0,t.jsx)(o.pre,{children:(0,t.jsx)(o.code,{children:'java -classpath "/my/druid/lib/*" org.apache.druid.cli.Main tools pull-deps --clean -c org.apache.druid.extensions:mysql-metadata-storage:33.0.0 -h org.apache.hadoop:hadoop-client:2.3.0 -h org.apache.hadoop:hadoop-client:2.4.0\n'})}),"\n",(0,t.jsxs)(o.p,{children:["Because ",(0,t.jsx)(o.code,{children:"--clean"})," is supplied, this command will first remove the directories specified at ",(0,t.jsx)(o.code,{children:"druid.extensions.directory"})," and ",(0,t.jsx)(o.code,{children:"druid.extensions.hadoopDependenciesDir"}),", then recreate them and start downloading the extensions there. After finishing downloading, if you go to the extension directories you specified, you will see"]}),"\n",(0,t.jsx)(o.pre,{children:(0,t.jsx)(o.code,{children:"tree extensions\nextensions\n\u2514\u2500\u2500 mysql-metadata-storage\n \u2514\u2500\u2500 mysql-metadata-storage-33.0.0.jar\n"})}),"\n",(0,t.jsx)(o.pre,{children:(0,t.jsx)(o.code,{children:"tree hadoop-dependencies\nhadoop-dependencies/\n\u2514\u2500\u2500 hadoop-client\n \u251c\u2500\u2500 2.3.0\n \u2502\xa0\xa0 \u251c\u2500\u2500 activation-1.1.jar\n \u2502\xa0\xa0 \u251c\u2500\u2500 avro-1.7.4.jar\n \u2502\xa0\xa0 \u251c\u2500\u2500 commons-beanutils-1.7.0.jar\n \u2502\xa0\xa0 \u251c\u2500\u2500 commons-beanutils-core-1.8.0.jar\n \u2502\xa0\xa0 \u251c\u2500\u2500 commons-cli-1.2.jar\n \u2502\xa0\xa0 \u251c\u2500\u2500 commons-codec-1.4.jar\n ..... lots of jars\n \u2514\u2500\u2500 2.4.0\n \u251c\u2500\u2500 activation-1.1.jar\n \u251c\u2500\u2500 avro-1.7.4.jar\n \u251c\u2500\u2500 commons-beanutils-1.7.0.jar\n \u251c\u2500\u2500 commons-beanutils-core-1.8.0.jar\n \u251c\u2500\u2500 commons-cli-1.2.jar\n \u251c\u2500\u2500 commons-codec-1.4.jar\n ..... lots of jars\n"})}),"\n",(0,t.jsxs)(o.p,{children:["Note that if you specify ",(0,t.jsx)(o.code,{children:"--defaultVersion"}),", you don't have to put version information in the coordinate. For example, if you want ",(0,t.jsx)(o.code,{children:"mysql-metadata-storage"})," to use version ",(0,t.jsx)(o.code,{children:"33.0.0"}),", you can change the command above to"]}),"\n",(0,t.jsx)(o.pre,{children:(0,t.jsx)(o.code,{children:'java -classpath "/my/druid/lib/*" org.apache.druid.cli.Main tools pull-deps --defaultVersion 33.0.0 --clean -c org.apache.druid.extensions:mysql-metadata-storage -h org.apache.hadoop:hadoop-client:2.3.0 -h org.apache.hadoop:hadoop-client:2.4.0\n'})}),"\n",(0,t.jsxs)(o.admonition,{type:"info",children:[(0,t.jsx)(o.p,{children:"Please note to use the pull-deps tool you must know the Maven groupId, artifactId, and version of your extension."}),(0,t.jsxs)(o.p,{children:["For Druid community extensions listed ",(0,t.jsx)(o.a,{href:"/docs/latest/configuration/extensions",children:"here"}),', the groupId is "org.apache.druid.extensions.contrib" and the artifactId is the name of the extension.']})]})]})}function p(e={}){const{wrapper:o}={...(0,d.R)(),...e.components};return o?(0,t.jsx)(o,{...e,children:(0,t.jsx)(l,{...e})}):l(e)}}}]); |