| "use strict";(self.webpackChunkwebsite=self.webpackChunkwebsite||[]).push([[1979],{3905:function(e,n,t){t.d(n,{Zo:function(){return u},kt:function(){return m}});var r=t(7294);function o(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function l(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);n&&(r=r.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,r)}return t}function a(e){for(var n=1;n<arguments.length;n++){var t=null!=arguments[n]?arguments[n]:{};n%2?l(Object(t),!0).forEach((function(n){o(e,n,t[n])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(t)):l(Object(t)).forEach((function(n){Object.defineProperty(e,n,Object.getOwnPropertyDescriptor(t,n))}))}return e}function i(e,n){if(null==e)return{};var t,r,o=function(e,n){if(null==e)return{};var t,r,o={},l=Object.keys(e);for(r=0;r<l.length;r++)t=l[r],n.indexOf(t)>=0||(o[t]=e[t]);return o}(e,n);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(r=0;r<l.length;r++)t=l[r],n.indexOf(t)>=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(o[t]=e[t])}return o}var p=r.createContext({}),s=function(e){var n=r.useContext(p),t=n;return e&&(t="function"==typeof e?e(n):a(a({},n),e)),t},u=function(e){var n=s(e.components);return r.createElement(p.Provider,{value:n},e.children)},c={inlineCode:"code",wrapper:function(e){var n=e.children;return r.createElement(r.Fragment,{},n)}},d=r.forwardRef((function(e,n){var t=e.components,o=e.mdxType,l=e.originalType,p=e.parentName,u=i(e,["components","mdxType","originalType","parentName"]),d=s(t),m=o,f=d["".concat(p,".").concat(m)]||d[m]||c[m]||l;return t?r.createElement(f,a(a({ref:n},u),{},{components:t})):r.createElement(f,a({ref:n},u))}));function m(e,n){var t=arguments,o=n&&n.mdxType;if("string"==typeof e||o){var l=t.length,a=new Array(l);a[0]=d;var i={};for(var p in n)hasOwnProperty.call(n,p)&&(i[p]=n[p]);i.originalType=e,i.mdxType="string"==typeof e?e:o,a[1]=i;for(var s=2;s<l;s++)a[s]=t[s];return r.createElement.apply(null,a)}return r.createElement.apply(null,t)}d.displayName="MDXCreateElement"},1664:function(e,n,t){t.r(n),t.d(n,{assets:function(){return u},contentTitle:function(){return p},default:function(){return m},frontMatter:function(){return i},metadata:function(){return s},toc:function(){return c}});var r=t(7462),o=t(3366),l=(t(7294),t(3905)),a=["components"],i={title:"Docker Images for TensorFlow"},p=void 0,s={unversionedId:"userDocs/yarn/WriteDockerfileTF",id:"version-0.6.0/userDocs/yarn/WriteDockerfileTF",title:"Docker Images for TensorFlow",description:"\x3c!--",source:"@site/versioned_docs/version-0.6.0/userDocs/yarn/WriteDockerfileTF.md",sourceDirName:"userDocs/yarn",slug:"/userDocs/yarn/WriteDockerfileTF",permalink:"/docs/0.6.0/userDocs/yarn/WriteDockerfileTF",editUrl:"https://github.com/apache/submarine/edit/master/website/versioned_docs/version-0.6.0/userDocs/yarn/WriteDockerfileTF.md",tags:[],version:"0.6.0",frontMatter:{title:"Docker Images for TensorFlow"}},u={},c=[{value:"How to create docker images to run Tensorflow on YARN",id:"how-to-create-docker-images-to-run-tensorflow-on-yarn",level:2},{value:"Use examples to build your own Tensorflow docker images",id:"use-examples-to-build-your-own-tensorflow-docker-images",level:2},{value:"Build Docker images",id:"build-docker-images",level:2},{value:"Manually build Docker image:",id:"manually-build-docker-image",level:3},{value:"Use prebuilt images",id:"use-prebuilt-images",level:3}],d={toc:c};function m(e){var n=e.components,t=(0,o.Z)(e,a);return(0,l.kt)("wrapper",(0,r.Z)({},d,t,{components:n,mdxType:"MDXLayout"}),(0,l.kt)("h2",{id:"how-to-create-docker-images-to-run-tensorflow-on-yarn"},"How to create docker images to run Tensorflow on YARN"),(0,l.kt)("p",null,"Dockerfile to run Tensorflow on YARN need two part:"),(0,l.kt)("p",null,(0,l.kt)("strong",{parentName:"p"},"Base libraries which Tensorflow depends on")),(0,l.kt)("p",null,"1) OS base image, for example ",(0,l.kt)("inlineCode",{parentName:"p"},"ubuntu:18.04")),(0,l.kt)("p",null,"2) Tensorflow depended libraries and packages. For example ",(0,l.kt)("inlineCode",{parentName:"p"},"python"),", ",(0,l.kt)("inlineCode",{parentName:"p"},"scipy"),". For GPU support, need ",(0,l.kt)("inlineCode",{parentName:"p"},"cuda"),", ",(0,l.kt)("inlineCode",{parentName:"p"},"cudnn"),", etc."),(0,l.kt)("p",null,"3) Tensorflow package."),(0,l.kt)("p",null,(0,l.kt)("strong",{parentName:"p"},"Libraries to access HDFS")),(0,l.kt)("p",null,"1) JDK"),(0,l.kt)("p",null,"2) Hadoop"),(0,l.kt)("p",null,"Here's an example of a base image (w/o GPU support) to install Tensorflow:"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-shell"},"FROM ubuntu:18.04\n\n# Pick up some TF dependencies\nRUN apt-get update && apt-get install -y --no-install-recommends \\\n build-essential \\\n curl \\\n libfreetype6-dev \\\n libpng-dev \\\n libzmq3-dev \\\n pkg-config \\\n python \\\n python-dev \\\n rsync \\\n software-properties-common \\\n unzip \\\n && \\\n apt-get clean && \\\n rm -rf /var/lib/apt/lists/*\n\nRUN export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -yq krb5-user libpam-krb5 && apt-get clean\n\nRUN curl -O https://bootstrap.pypa.io/get-pip.py && \\\n python get-pip.py && \\\n rm get-pip.py\n\nRUN pip --no-cache-dir install \\\n Pillow \\\n h5py \\\n ipykernel \\\n jupyter \\\n matplotlib \\\n numpy \\\n pandas \\\n scipy \\\n sklearn \\\n && \\\n python -m ipykernel.kernelspec\n\nRUN pip --no-cache-dir install \\\n http://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.13.1-cp27-none-linux_x86_64.whl\n")),(0,l.kt)("p",null,"On top of above image, add files, install packages to access HDFS"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-shell"},'RUN apt-get update && apt-get install -y openjdk-8-jdk wget\n# Install hadoop\nENV HADOOP_VERSION="2.9.2"\nRUN wget http://mirrors.hust.edu.cn/apache/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz\nRUN tar zxf hadoop-${HADOOP_VERSION}.tar.gz\nRUN ln -s hadoop-${HADOOP_VERSION} hadoop-current\nRUN rm hadoop-${HADOOP_VERSION}.tar.gz\n')),(0,l.kt)("p",null,"Build and push to your own docker registry: Use ",(0,l.kt)("inlineCode",{parentName:"p"},"docker build ... ")," and ",(0,l.kt)("inlineCode",{parentName:"p"},"docker push ...")," to finish this step."),(0,l.kt)("h2",{id:"use-examples-to-build-your-own-tensorflow-docker-images"},"Use examples to build your own Tensorflow docker images"),(0,l.kt)("p",null,"We provided following examples for you to build tensorflow docker images."),(0,l.kt)("p",null,"For Tensorflow 1.13.1 (Precompiled to CUDA 10.x)"),(0,l.kt)("ul",null,(0,l.kt)("li",{parentName:"ul"},(0,l.kt)("em",{parentName:"li"},"docker/tensorflow/base/ubuntu-18.04/Dockerfile.cpu.tf_1.13.1"),": Tensorflow 1.13.1 supports CPU only."),(0,l.kt)("li",{parentName:"ul"},(0,l.kt)("em",{parentName:"li"},"docker/tensorflow/with-cifar10-models/ubuntu-18.04/Dockerfile.cpu.tf_1.13.1"),": Tensorflow 1.13.1 supports CPU only, and included models"),(0,l.kt)("li",{parentName:"ul"},(0,l.kt)("em",{parentName:"li"},"docker/tensorflow/base/ubuntu-18.04/Dockerfile.gpu.tf_1.13.1"),": Tensorflow 1.13.1 supports GPU, which is prebuilt to CUDA10."),(0,l.kt)("li",{parentName:"ul"},(0,l.kt)("em",{parentName:"li"},"docker/tensorflow/with-cifar10-models/ubuntu-18.04/Dockerfile.gpu.tf_1.13.1"),": Tensorflow 1.13.1 supports GPU, which is prebuilt to CUDA10, with models.")),(0,l.kt)("h2",{id:"build-docker-images"},"Build Docker images"),(0,l.kt)("h3",{id:"manually-build-docker-image"},"Manually build Docker image:"),(0,l.kt)("p",null,"Under ",(0,l.kt)("inlineCode",{parentName:"p"},"docker/")," directory, run ",(0,l.kt)("inlineCode",{parentName:"p"},"build-all.sh")," to build Docker images. It will build following images:"),(0,l.kt)("ul",null,(0,l.kt)("li",{parentName:"ul"},(0,l.kt)("inlineCode",{parentName:"li"},"tf-1.13.1-gpu-base:0.0.1")," for base Docker image which includes Hadoop, Tensorflow, GPU base libraries."),(0,l.kt)("li",{parentName:"ul"},(0,l.kt)("inlineCode",{parentName:"li"},"tf-1.13.1-gpu-base:0.0.1")," for base Docker image which includes Hadoop. Tensorflow."),(0,l.kt)("li",{parentName:"ul"},(0,l.kt)("inlineCode",{parentName:"li"},"tf-1.13.1-gpu:0.0.1")," which includes cifar10 model"),(0,l.kt)("li",{parentName:"ul"},(0,l.kt)("inlineCode",{parentName:"li"},"tf-1.13.1-cpu:0.0.1")," which inclues cifar10 model (cpu only).")),(0,l.kt)("h3",{id:"use-prebuilt-images"},"Use prebuilt images"),(0,l.kt)("p",null,"(No liability)\nYou can also use prebuilt images for convenience:"),(0,l.kt)("ul",null,(0,l.kt)("li",{parentName:"ul"},"hadoopsubmarine/tf-1.13.1-gpu:0.0.1"),(0,l.kt)("li",{parentName:"ul"},"hadoopsubmarine/tf-1.13.1-cpu:0.0.1")))}m.isMDXComponent=!0}}]); |