| "use strict";(self.webpackChunkwebsite=self.webpackChunkwebsite||[]).push([[4748],{3905:function(e,n,r){r.d(n,{Zo:function(){return l},kt:function(){return m}});var t=r(7294);function o(e,n,r){return n in e?Object.defineProperty(e,n,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[n]=r,e}function a(e,n){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var t=Object.getOwnPropertySymbols(e);n&&(t=t.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),r.push.apply(r,t)}return r}function i(e){for(var n=1;n<arguments.length;n++){var r=null!=arguments[n]?arguments[n]:{};n%2?a(Object(r),!0).forEach((function(n){o(e,n,r[n])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(r)):a(Object(r)).forEach((function(n){Object.defineProperty(e,n,Object.getOwnPropertyDescriptor(r,n))}))}return e}function u(e,n){if(null==e)return{};var r,t,o=function(e,n){if(null==e)return{};var r,t,o={},a=Object.keys(e);for(t=0;t<a.length;t++)r=a[t],n.indexOf(r)>=0||(o[r]=e[r]);return o}(e,n);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(t=0;t<a.length;t++)r=a[t],n.indexOf(r)>=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(o[r]=e[r])}return o}var s=t.createContext({}),c=function(e){var n=t.useContext(s),r=n;return e&&(r="function"==typeof e?e(n):i(i({},n),e)),r},l=function(e){var n=c(e.components);return t.createElement(s.Provider,{value:n},e.children)},p={inlineCode:"code",wrapper:function(e){var n=e.children;return t.createElement(t.Fragment,{},n)}},d=t.forwardRef((function(e,n){var r=e.components,o=e.mdxType,a=e.originalType,s=e.parentName,l=u(e,["components","mdxType","originalType","parentName"]),d=c(r),m=o,f=d["".concat(s,".").concat(m)]||d[m]||p[m]||a;return r?t.createElement(f,i(i({ref:n},l),{},{components:r})):t.createElement(f,i({ref:n},l))}));function m(e,n){var r=arguments,o=n&&n.mdxType;if("string"==typeof e||o){var a=r.length,i=new Array(a);i[0]=d;var u={};for(var s in n)hasOwnProperty.call(n,s)&&(u[s]=n[s]);u.originalType=e,u.mdxType="string"==typeof e?e:o,i[1]=u;for(var c=2;c<a;c++)i[c]=r[c];return t.createElement.apply(null,i)}return t.createElement.apply(null,r)}d.displayName="MDXCreateElement"},9085:function(e,n,r){r.r(n),r.d(n,{assets:function(){return l},contentTitle:function(){return s},default:function(){return m},frontMatter:function(){return u},metadata:function(){return c},toc:function(){return p}});var t=r(7462),o=r(3366),a=(r(7294),r(3905)),i=["components"],u={title:"Running Submarine on YARN"},s=void 0,c={unversionedId:"adminDocs/yarn/README",id:"version-0.6.0/adminDocs/yarn/README",title:"Running Submarine on YARN",description:"\x3c!--",source:"@site/versioned_docs/version-0.6.0/adminDocs/yarn/README.md",sourceDirName:"adminDocs/yarn",slug:"/adminDocs/yarn/",permalink:"/docs/0.6.0/adminDocs/yarn/",editUrl:"https://github.com/apache/submarine/edit/master/website/versioned_docs/version-0.6.0/adminDocs/yarn/README.md",tags:[],version:"0.6.0",frontMatter:{title:"Running Submarine on YARN"},sidebar:"docs",previous:{title:"Tensorboard",permalink:"/docs/0.6.0/userDocs/others/tensorboard"},next:{title:"Project Architecture",permalink:"/docs/0.6.0/devDocs/"}},l={},p=[{value:"Hadoop version",id:"hadoop-version",level:2},{value:"Submarine YARN Runtime Guide",id:"submarine-yarn-runtime-guide",level:2}],d={toc:p};function m(e){var n=e.components,r=(0,o.Z)(e,i);return(0,a.kt)("wrapper",(0,t.Z)({},d,r,{components:n,mdxType:"MDXLayout"}),(0,a.kt)("p",null,"Submarine for YARN supports TensorFlow, PyTorch and MXNet framework. (Which is leveraging ",(0,a.kt)("a",{parentName:"p",href:"https://github.com/linkedin/TonY"},"TonY")," created by Linkedin to run deep learning training jobs on YARN."),(0,a.kt)("p",null,"Submarine also supports GPU-on-YARN and Docker-on-YARN feature."),(0,a.kt)("p",null,"Submarine can run on Hadoop 2.7.3 or later version, if GPU-on-YARN or Docker-on-YARN feature is needed, newer Hadoop version is required, please refer to the next section about what Hadoop version to choose."),(0,a.kt)("h2",{id:"hadoop-version"},"Hadoop version"),(0,a.kt)("p",null,"Must:"),(0,a.kt)("ul",null,(0,a.kt)("li",{parentName:"ul"},"Apache Hadoop version newer than 2.7.3")),(0,a.kt)("p",null,"Optional:"),(0,a.kt)("ul",null,(0,a.kt)("li",{parentName:"ul"},"When you want to use GPU-on-YARN feature with Submarine, please make sure Hadoop is at least 2.10.0+ (or 3.1.0+), and follow ",(0,a.kt)("a",{parentName:"li",href:"https://hadoop.apache.org/docs/r2.10.0/hadoop-yarn/hadoop-yarn-site/UsingGpus.html"},"Enable GPU on YARN 2.10.0+")," to enable GPU-on-YARN feature."),(0,a.kt)("li",{parentName:"ul"},"When you want to run training jobs with Docker container, please make sure Hadoop is at least 2.8.2, and follow ",(0,a.kt)("a",{parentName:"li",href:"https://hadoop.apache.org/docs/r2.8.2/hadoop-yarn/hadoop-yarn-site/DockerContainers.html"},"Enable Docker on YARN 2.8.2+")," to enable Docker-on-YARN feature.")),(0,a.kt)("h2",{id:"submarine-yarn-runtime-guide"},"Submarine YARN Runtime Guide"),(0,a.kt)("p",null,(0,a.kt)("a",{parentName:"p",href:"../../userDocs/yarn/YARNRuntimeGuide"},"YARN Runtime Guide")," talk about how to use Submarine to run jobs on YARN, with Docker / without Docker."))}m.isMDXComponent=!0}}]); |