| (window.webpackJsonp=window.webpackJsonp||[]).push([[33],{100:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return u}));var r=n(3),o=n(7),a=(n(0),n(144)),i={title:"Docker Images for PyTorch"},c={unversionedId:"userDocs/yarn/WriteDockerfilePT",id:"userDocs/yarn/WriteDockerfilePT",isDocsHomePage:!1,title:"Docker Images for PyTorch",description:"\x3c!--",source:"@site/docs/userDocs/yarn/WriteDockerfilePT.md",slug:"/userDocs/yarn/WriteDockerfilePT",permalink:"/docs/userDocs/yarn/WriteDockerfilePT",editUrl:"https://github.com/apache/submarine/edit/master/website/docs/userDocs/yarn/WriteDockerfilePT.md",version:"current"},l=[{value:"How to create docker images to run PyTorch on YARN",id:"how-to-create-docker-images-to-run-pytorch-on-yarn",children:[]},{value:"Use examples to build your own PyTorch docker images",id:"use-examples-to-build-your-own-pytorch-docker-images",children:[]},{value:"Build Docker images",id:"build-docker-images",children:[{value:"Manually build Docker image:",id:"manually-build-docker-image",children:[]},{value:"Use prebuilt images",id:"use-prebuilt-images",children:[]}]}],p={toc:l};function u(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h2",{id:"how-to-create-docker-images-to-run-pytorch-on-yarn"},"How to create docker images to run PyTorch on YARN"),Object(a.b)("p",null,"Dockerfile to run PyTorch on YARN needs two parts:"),Object(a.b)("p",null,Object(a.b)("strong",{parentName:"p"},"Base libraries which PyTorch depends on")),Object(a.b)("p",null,"1) OS base image, for example ",Object(a.b)("inlineCode",{parentName:"p"},"ubuntu:18.04")),Object(a.b)("p",null,"2) PyTorch dependent libraries and packages. For example ",Object(a.b)("inlineCode",{parentName:"p"},"python"),", ",Object(a.b)("inlineCode",{parentName:"p"},"scipy"),". For GPU support, you also need ",Object(a.b)("inlineCode",{parentName:"p"},"cuda"),", ",Object(a.b)("inlineCode",{parentName:"p"},"cudnn"),", etc."),Object(a.b)("p",null,"3) PyTorch package."),Object(a.b)("p",null,Object(a.b)("strong",{parentName:"p"},"Libraries to access HDFS")),Object(a.b)("p",null,"1) JDK"),Object(a.b)("p",null,"2) Hadoop"),Object(a.b)("p",null,"Here's an example of a base image (with GPU support) to install PyTorch:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-shell"},'FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04\nARG PYTHON_VERSION=3.6\nRUN apt-get update && apt-get install -y --no-install-recommends \\\n build-essential \\\n cmake \\\n git \\\n curl \\\n vim \\\n ca-certificates \\\n libjpeg-dev \\\n libpng-dev \\\n wget &&\\\n rm -rf /var/lib/apt/lists/*\n\n\nRUN curl -o ~/miniconda.sh -O https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \\\n chmod +x ~/miniconda.sh && \\\n ~/miniconda.sh -b -p /opt/conda && \\\n rm ~/miniconda.sh && \\\n /opt/conda/bin/conda install -y python=$PYTHON_VERSION numpy pyyaml scipy ipython mkl mkl-include cython typing && \\\n /opt/conda/bin/conda install -y -c pytorch magma-cuda100 && \\\n /opt/conda/bin/conda clean -ya\nENV PATH /opt/conda/bin:$PATH\nRUN pip install ninja\n# This must be done before pip so that requirements.txt is available\nWORKDIR /opt/pytorch\nRUN git clone https://github.com/pytorch/pytorch.git\nWORKDIR pytorch\nRUN git submodule update --init\nRUN TORCH_CUDA_ARCH_LIST="3.5 5.2 6.0 6.1 7.0+PTX" TORCH_NVCC_FLAGS="-Xfatbin -compress-all" \\\n CMAKE_PREFIX_PATH="$(dirname $(which conda))/../" \\\n pip install -v .\n\nWORKDIR /opt/pytorch\nRUN git clone https://github.com/pytorch/vision.git && cd vision && pip install -v .\n\n')),Object(a.b)("p",null,"On top of above image, add files, install packages to access HDFS"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-shell"},'RUN apt-get update && apt-get install -y openjdk-8-jdk wget\n# Install hadoop\nENV HADOOP_VERSION="2.9.2"\nRUN wget http://mirrors.hust.edu.cn/apache/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz\nRUN tar zxf hadoop-${HADOOP_VERSION}.tar.gz\nRUN ln -s hadoop-${HADOOP_VERSION} hadoop-current\nRUN rm hadoop-${HADOOP_VERSION}.tar.gz\n')),Object(a.b)("p",null,"Build and push to your own docker registry: Use ",Object(a.b)("inlineCode",{parentName:"p"},"docker build ... ")," and ",Object(a.b)("inlineCode",{parentName:"p"},"docker push ...")," to finish this step."),Object(a.b)("h2",{id:"use-examples-to-build-your-own-pytorch-docker-images"},"Use examples to build your own PyTorch docker images"),Object(a.b)("p",null,"We provided some example Dockerfiles for you to build your own PyTorch docker images."),Object(a.b)("p",null,"For latest PyTorch"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},Object(a.b)("em",{parentName:"li"},"docker/pytorch/base/ubuntu-18.04/Dockerfile.gpu.pytorch_latest"),": Latest Pytorch that supports GPU, which is prebuilt to CUDA10."),Object(a.b)("li",{parentName:"ul"},Object(a.b)("em",{parentName:"li"},"docker/pytorch/with-cifar10-models/ubuntu-18.04/Dockerfile.gpu.pytorch_latest"),": Latest Pytorch that GPU, which is prebuilt to CUDA10, with models.")),Object(a.b)("h2",{id:"build-docker-images"},"Build Docker images"),Object(a.b)("h3",{id:"manually-build-docker-image"},"Manually build Docker image:"),Object(a.b)("p",null,"Under ",Object(a.b)("inlineCode",{parentName:"p"},"docker/pytorch")," directory, run ",Object(a.b)("inlineCode",{parentName:"p"},"build-all.sh")," to build all Docker images. This command will build the following Docker images:"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},Object(a.b)("inlineCode",{parentName:"li"},"pytorch-latest-gpu-base:0.0.1")," for base Docker image which includes Hadoop, PyTorch, GPU base libraries."),Object(a.b)("li",{parentName:"ul"},Object(a.b)("inlineCode",{parentName:"li"},"pytorch-latest-gpu:0.0.1")," which includes cifar10 model as well")),Object(a.b)("h3",{id:"use-prebuilt-images"},"Use prebuilt images"),Object(a.b)("p",null,"(No liability)\nYou can also use prebuilt images for convenience:"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"hadoopsubmarine/pytorch-latest-gpu-base:0.0.1")))}u.isMDXComponent=!0},144:function(e,t,n){"use strict";n.d(t,"a",(function(){return s})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var p=o.a.createContext({}),u=function(e){var t=o.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},s=function(e){var t=u(e.components);return o.a.createElement(p.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},d=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,p=l(e,["components","mdxType","originalType","parentName"]),s=u(n),d=r,m=s["".concat(i,".").concat(d)]||s[d]||b[d]||a;return n?o.a.createElement(m,c(c({ref:t},p),{},{components:n})):o.a.createElement(m,c({ref:t},p))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=d;var c={};for(var l in t)hasOwnProperty.call(t,l)&&(c[l]=t[l]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var p=2;p<a;p++)i[p]=n[p];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]); |