| "use strict";(self.webpackChunkwebsite=self.webpackChunkwebsite||[]).push([[4457],{3905:function(e,t,n){n.d(t,{Zo:function(){return u},kt:function(){return m}});var r=n(7294);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function i(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=r.createContext({}),p=function(e){var t=r.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},u=function(e){var t=p(e.components);return r.createElement(c.Provider,{value:t},e.children)},s={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},d=r.forwardRef((function(e,t){var n=e.components,o=e.mdxType,a=e.originalType,c=e.parentName,u=l(e,["components","mdxType","originalType","parentName"]),d=p(n),m=o,h=d["".concat(c,".").concat(m)]||d[m]||s[m]||a;return n?r.createElement(h,i(i({ref:t},u),{},{components:n})):r.createElement(h,i({ref:t},u))}));function m(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=n.length,i=new Array(a);i[0]=d;var l={};for(var c in t)hasOwnProperty.call(t,c)&&(l[c]=t[c]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var p=2;p<a;p++)i[p]=n[p];return r.createElement.apply(null,i)}return r.createElement.apply(null,n)}d.displayName="MDXCreateElement"},259:function(e,t,n){n.r(t),n.d(t,{assets:function(){return u},contentTitle:function(){return c},default:function(){return m},frontMatter:function(){return l},metadata:function(){return p},toc:function(){return s}});var r=n(7462),o=n(3366),a=(n(7294),n(3905)),i=["components"],l={title:"Docker Images for PyTorch"},c=void 0,p={unversionedId:"userDocs/yarn/WriteDockerfilePT",id:"version-0.6.0/userDocs/yarn/WriteDockerfilePT",title:"Docker Images for PyTorch",description:"\x3c!--",source:"@site/versioned_docs/version-0.6.0/userDocs/yarn/WriteDockerfilePT.md",sourceDirName:"userDocs/yarn",slug:"/userDocs/yarn/WriteDockerfilePT",permalink:"/docs/userDocs/yarn/WriteDockerfilePT",editUrl:"https://github.com/apache/submarine/edit/master/website/versioned_docs/version-0.6.0/userDocs/yarn/WriteDockerfilePT.md",tags:[],version:"0.6.0",frontMatter:{title:"Docker Images for PyTorch"}},u={},s=[{value:"How to create docker images to run PyTorch on YARN",id:"how-to-create-docker-images-to-run-pytorch-on-yarn",level:2},{value:"Use examples to build your own PyTorch docker images",id:"use-examples-to-build-your-own-pytorch-docker-images",level:2},{value:"Build Docker images",id:"build-docker-images",level:2},{value:"Manually build Docker image:",id:"manually-build-docker-image",level:3},{value:"Use prebuilt images",id:"use-prebuilt-images",level:3}],d={toc:s};function m(e){var t=e.components,n=(0,o.Z)(e,i);return(0,a.kt)("wrapper",(0,r.Z)({},d,n,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("h2",{id:"how-to-create-docker-images-to-run-pytorch-on-yarn"},"How to create docker images to run PyTorch on YARN"),(0,a.kt)("p",null,"Dockerfile to run PyTorch on YARN needs two parts:"),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"Base libraries which PyTorch depends on")),(0,a.kt)("p",null,"1) OS base image, for example ",(0,a.kt)("inlineCode",{parentName:"p"},"ubuntu:18.04")),(0,a.kt)("p",null,"2) PyTorch dependent libraries and packages. For example ",(0,a.kt)("inlineCode",{parentName:"p"},"python"),", ",(0,a.kt)("inlineCode",{parentName:"p"},"scipy"),". For GPU support, you also need ",(0,a.kt)("inlineCode",{parentName:"p"},"cuda"),", ",(0,a.kt)("inlineCode",{parentName:"p"},"cudnn"),", etc."),(0,a.kt)("p",null,"3) PyTorch package."),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"Libraries to access HDFS")),(0,a.kt)("p",null,"1) JDK"),(0,a.kt)("p",null,"2) Hadoop"),(0,a.kt)("p",null,"Here's an example of a base image (with GPU support) to install PyTorch:"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},'FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04\nARG PYTHON_VERSION=3.6\nRUN apt-get update && apt-get install -y --no-install-recommends \\\n build-essential \\\n cmake \\\n git \\\n curl \\\n vim \\\n ca-certificates \\\n libjpeg-dev \\\n libpng-dev \\\n wget &&\\\n rm -rf /var/lib/apt/lists/*\n\n\nRUN curl -o ~/miniconda.sh -O https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \\\n chmod +x ~/miniconda.sh && \\\n ~/miniconda.sh -b -p /opt/conda && \\\n rm ~/miniconda.sh && \\\n /opt/conda/bin/conda install -y python=$PYTHON_VERSION numpy pyyaml scipy ipython mkl mkl-include cython typing && \\\n /opt/conda/bin/conda install -y -c pytorch magma-cuda100 && \\\n /opt/conda/bin/conda clean -ya\nENV PATH /opt/conda/bin:$PATH\nRUN pip install ninja\n# This must be done before pip so that requirements.txt is available\nWORKDIR /opt/pytorch\nRUN git clone https://github.com/pytorch/pytorch.git\nWORKDIR pytorch\nRUN git submodule update --init\nRUN TORCH_CUDA_ARCH_LIST="3.5 5.2 6.0 6.1 7.0+PTX" TORCH_NVCC_FLAGS="-Xfatbin -compress-all" \\\n CMAKE_PREFIX_PATH="$(dirname $(which conda))/../" \\\n pip install -v .\n\nWORKDIR /opt/pytorch\nRUN git clone https://github.com/pytorch/vision.git && cd vision && pip install -v .\n\n')),(0,a.kt)("p",null,"On top of above image, add files, install packages to access HDFS"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},'RUN apt-get update && apt-get install -y openjdk-8-jdk wget\n# Install hadoop\nENV HADOOP_VERSION="2.9.2"\nRUN wget http://mirrors.hust.edu.cn/apache/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz\nRUN tar zxf hadoop-${HADOOP_VERSION}.tar.gz\nRUN ln -s hadoop-${HADOOP_VERSION} hadoop-current\nRUN rm hadoop-${HADOOP_VERSION}.tar.gz\n')),(0,a.kt)("p",null,"Build and push to your own docker registry: Use ",(0,a.kt)("inlineCode",{parentName:"p"},"docker build ... ")," and ",(0,a.kt)("inlineCode",{parentName:"p"},"docker push ...")," to finish this step."),(0,a.kt)("h2",{id:"use-examples-to-build-your-own-pytorch-docker-images"},"Use examples to build your own PyTorch docker images"),(0,a.kt)("p",null,"We provided some example Dockerfiles for you to build your own PyTorch docker images."),(0,a.kt)("p",null,"For latest PyTorch"),(0,a.kt)("ul",null,(0,a.kt)("li",{parentName:"ul"},(0,a.kt)("em",{parentName:"li"},"docker/pytorch/base/ubuntu-18.04/Dockerfile.gpu.pytorch_latest"),": Latest Pytorch that supports GPU, which is prebuilt to CUDA10."),(0,a.kt)("li",{parentName:"ul"},(0,a.kt)("em",{parentName:"li"},"docker/pytorch/with-cifar10-models/ubuntu-18.04/Dockerfile.gpu.pytorch_latest"),": Latest Pytorch that GPU, which is prebuilt to CUDA10, with models.")),(0,a.kt)("h2",{id:"build-docker-images"},"Build Docker images"),(0,a.kt)("h3",{id:"manually-build-docker-image"},"Manually build Docker image:"),(0,a.kt)("p",null,"Under ",(0,a.kt)("inlineCode",{parentName:"p"},"docker/pytorch")," directory, run ",(0,a.kt)("inlineCode",{parentName:"p"},"build-all.sh")," to build all Docker images. This command will build the following Docker images:"),(0,a.kt)("ul",null,(0,a.kt)("li",{parentName:"ul"},(0,a.kt)("inlineCode",{parentName:"li"},"pytorch-latest-gpu-base:0.0.1")," for base Docker image which includes Hadoop, PyTorch, GPU base libraries."),(0,a.kt)("li",{parentName:"ul"},(0,a.kt)("inlineCode",{parentName:"li"},"pytorch-latest-gpu:0.0.1")," which includes cifar10 model as well")),(0,a.kt)("h3",{id:"use-prebuilt-images"},"Use prebuilt images"),(0,a.kt)("p",null,"(No liability)\nYou can also use prebuilt images for convenience:"),(0,a.kt)("ul",null,(0,a.kt)("li",{parentName:"ul"},"hadoopsubmarine/pytorch-latest-gpu-base:0.0.1")))}m.isMDXComponent=!0}}]); |