(window.webpackJsonp=window.webpackJsonp||[]).push([[38],{105:function(e,t,a){"use strict";a.r(t),a.d(t,"frontMatter",(function(){return l})),a.d(t,"metadata",(function(){return i})),a.d(t,"toc",(function(){return o})),a.d(t,"default",(function(){return s}));var n=a(3),r=a(7),b=(a(0),a(144)),l={title:"Development Guide"},i={unversionedId:"devDocs/Development",id:"devDocs/Development",isDocsHomePage:!1,title:"Development Guide",description:"\x3c!--",source:"@site/docs/devDocs/Development.md",slug:"/devDocs/Development",permalink:"/docs/devDocs/Development",editUrl:"https://github.com/apache/submarine/edit/master/website/docs/devDocs/Development.md",version:"current",sidebar:"docs",previous:{title:"How to Build Submarine",permalink:"/docs/devDocs/BuildFromCode"},next:{title:"How to Run Integration K8s Test",permalink:"/docs/devDocs/IntegrationTestK8s"}},o=[{value:"Video",id:"video",children:[]},{value:"Develop server",id:"develop-server",children:[{value:"Prerequisites",id:"prerequisites",children:[]},{value:"Setting up checkstyle in IDE",id:"setting-up-checkstyle-in-ide",children:[]},{value:"Testing",id:"testing",children:[]},{value:"Build from source",id:"build-from-source",children:[]}]},{value:"Develop workbench",id:"develop-workbench",children:[]},{value:"Develop database",id:"develop-database",children:[]},{value:"Develop Submarine Website",id:"develop-submarine-website",children:[{value:"Add a new page",id:"add-a-new-page",children:[]},{value:"Installation",id:"installation",children:[]},{value:"Build",id:"build",children:[]},{value:"Local Development",id:"local-development",children:[]}]}],c={toc:o};function s(e){var t=e.components,a=Object(r.a)(e,["components"]);return Object(b.b)("wrapper",Object(n.a)({},c,a,{components:t,mdxType:"MDXLayout"}),Object(b.b)("h1",{id:"project-overview"},"Project Overview"),Object(b.b)("p",null,"The document ",Object(b.b)("a",{parentName:"p",href:"/docs/"},"Submarine Local Deployment")," shows how to deploy the Submarine service to your Kubernetes cluster. The Submarine service consists mainly of nine components, and you can check them with the following command:"),Object(b.b)("pre",null,Object(b.b)("code",{parentName:"pre"},"kubectl get pods -n ${your_namespace}\n")),Object(b.b)("p",null,"A brief introduction about these components: "),Object(b.b)("ol",null,Object(b.b)("li",{parentName:"ol"},Object(b.b)("strong",{parentName:"li"},"tf-operator"),": Enable users to run TensorFlow jobs distributedly"),Object(b.b)("li",{parentName:"ol"},Object(b.b)("strong",{parentName:"li"},"pytorch-operator"),": Enable users to run PyTorch jobs distributedly"),Object(b.b)("li",{parentName:"ol"},Object(b.b)("strong",{parentName:"li"},"notebook-controller"),": Jupyter Notebook controller"),Object(b.b)("li",{parentName:"ol"},Object(b.b)("strong",{parentName:"li"},"submarine-traefik"),": Kubernetes Ingress controller"),Object(b.b)("li",{parentName:"ol"},Object(b.b)("strong",{parentName:"li"},"submarine-database"),": A MySQL database to store metadata"),Object(b.b)("li",{parentName:"ol"},Object(b.b)("strong",{parentName:"li"},"submarine-minio"),": An object store for machine learning artifacts"),Object(b.b)("li",{parentName:"ol"},Object(b.b)("strong",{parentName:"li"},"submarine-mlflow"),": A platform for model management"),Object(b.b)("li",{parentName:"ol"},Object(b.b)("strong",{parentName:"li"},"submarine-tensorboard"),": A visualization tool for distributed training experiments"),Object(b.b)("li",{parentName:"ol"},Object(b.b)("strong",{parentName:"li"},"submarine-server"),": Handle API requests, and submit distributed training experiments to Kubernetes.")),Object(b.b)("h1",{id:"submarine-development"},"Submarine Development"),Object(b.b)("h2",{id:"video"},"Video"),Object(b.b)("ul",null,Object(b.b)("li",{parentName:"ul"},"From this ",Object(b.b)("a",{parentName:"li",href:"https://youtu.be/32Na2k6Alv4"},"Video"),", you will know how to deal with the configuration of Submarine and be able to contribute to it via Github.")),Object(b.b)("h2",{id:"develop-server"},"Develop server"),Object(b.b)("h3",{id:"prerequisites"},"Prerequisites"),Object(b.b)("ul",null,Object(b.b)("li",{parentName:"ul"},"JDK 1.8"),Object(b.b)("li",{parentName:"ul"},"Maven 3.3 or later ( < 3.8.1 )"),Object(b.b)("li",{parentName:"ul"},"Docker")),Object(b.b)("h3",{id:"setting-up-checkstyle-in-ide"},"Setting up checkstyle in IDE"),Object(b.b)("p",null,"Checkstyle plugin may help to detect violations directly from the IDE."),Object(b.b)("ol",null,Object(b.b)("li",{parentName:"ol"},"Install Checkstyle+IDEA plugin from Preference -> Plugins"),Object(b.b)("li",{parentName:"ol"},"Open Preference -> Tools -> Checkstyle ->",Object(b.b)("ol",{parentName:"li"},Object(b.b)("li",{parentName:"ol"},"Set Checkstyle version:",Object(b.b)("ul",{parentName:"li"},Object(b.b)("li",{parentName:"ul"},"Checkstyle version: 8.0"))),Object(b.b)("li",{parentName:"ol"},"Add (+) a new Configuration File",Object(b.b)("ul",{parentName:"li"},Object(b.b)("li",{parentName:"ul"},"Description: Submarine"),Object(b.b)("li",{parentName:"ul"},"Use a local checkstyle ${SUBMARINE_HOME}/dev-support/maven-config/checkstyle.xml"))))),Object(b.b)("li",{parentName:"ol"},"Open the Checkstyle Tool Window, select the Submarine rule and execute the check")),Object(b.b)("h3",{id:"testing"},"Testing"),Object(b.b)("ul",null,Object(b.b)("li",{parentName:"ul"},Object(b.b)("p",{parentName:"li"},"Unit Test"),Object(b.b)("p",{parentName:"li"},"For each class, there is a corresponding testClass. For example, ",Object(b.b)("inlineCode",{parentName:"p"},"SubmarineServerTest")," is used for testing ",Object(b.b)("inlineCode",{parentName:"p"},"SubmarineServer"),". Whenever you add a funtion in classes, you must write a unit test to test it.")),Object(b.b)("li",{parentName:"ul"},Object(b.b)("p",{parentName:"li"},"Integration Test: ",Object(b.b)("a",{parentName:"p",href:"/docs/devDocs/IntegrationTestK8s"},"IntegrationTestK8s.md")))),Object(b.b)("h3",{id:"build-from-source"},"Build from source"),Object(b.b)("ul",null,Object(b.b)("li",{parentName:"ul"},Object(b.b)("p",{parentName:"li"},"Before building"),Object(b.b)("ol",{parentName:"li"},Object(b.b)("li",{parentName:"ol"},"We assume the developer use ",Object(b.b)("strong",{parentName:"li"},"minikube")," as a local kubernetes cluster."),Object(b.b)("li",{parentName:"ol"},"Make sure you have ",Object(b.b)("strong",{parentName:"li"},"installed the submarine helm-chart")," in the cluster.")))),Object(b.b)("ol",null,Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Package the Submarine server into a new jar file"),Object(b.b)("pre",{parentName:"li"},Object(b.b)("code",{parentName:"pre",className:"language-bash"},"mvn package -DskipTests\n"))),Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Build the new server docker image in minikube"),Object(b.b)("pre",{parentName:"li"},Object(b.b)("code",{parentName:"pre",className:"language-bash"},"# switch to minikube docker daemon to build image directly in minikube\neval $(minikube docker-env)\n\n# run docker build\n./dev-support/docker-images/submarine/build.sh\n\n# exit minikube docker daemon\neval $(minikube docker-env -u)\n"))),Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Update server pod"),Object(b.b)("pre",{parentName:"li"},Object(b.b)("code",{parentName:"pre",className:"language-bash"},"helm upgrade --set submarine.server.dev=true submarine ./helm-charts/submarine\n")),Object(b.b)("p",{parentName:"li"},"Set ",Object(b.b)("inlineCode",{parentName:"p"},"submarine.server.dev")," to ",Object(b.b)("inlineCode",{parentName:"p"},"true"),", enabling the server pod to be launched with the new docker image."))),Object(b.b)("h2",{id:"develop-workbench"},"Develop workbench"),Object(b.b)("ol",null,Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Deploy the Submarine"),Object(b.b)("p",{parentName:"li"},"Follow ",Object(b.b)("a",{parentName:"p",href:"/docs/"},"Getting Started/Submarine Local Deployment"),", and make sure you can connect to ",Object(b.b)("inlineCode",{parentName:"p"},"http://localhost:32080")," in the browser.")),Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Install the dependencies"),Object(b.b)("pre",{parentName:"li"},Object(b.b)("code",{parentName:"pre",className:"language-bash"},"cd submarine-workbench/workbench-web\nnpm install\n"))),Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Run the workbench based on proxy server"),Object(b.b)("pre",{parentName:"li"},Object(b.b)("code",{parentName:"pre",className:"language-bash"},"npm run start\n")),Object(b.b)("ol",{parentName:"li"},Object(b.b)("li",{parentName:"ol"},"The request sent to ",Object(b.b)("inlineCode",{parentName:"li"},"http://localhost:4200")," will be redirected to ",Object(b.b)("inlineCode",{parentName:"li"},"http://localhost:32080"),"."),Object(b.b)("li",{parentName:"ol"},"Open ",Object(b.b)("inlineCode",{parentName:"li"},"http://localhost:4200")," in browser to see the real-time change of workbench."))),Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Frontend E2E test: ",Object(b.b)("a",{parentName:"p",href:"/docs/devDocs/IntegrationTestE2E"},"IntegrationTestE2E.md")," "))),Object(b.b)("h2",{id:"develop-database"},"Develop database"),Object(b.b)("ol",null,Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Build the docker image"),Object(b.b)("pre",{parentName:"li"},Object(b.b)("code",{parentName:"pre",className:"language-bash"},"# switch to minikube docker daemon to build image directly in minikube\neval $(minikube docker-env)\n\n# run docker build\n./dev-support/docker-images/database/build.sh\n\n# exit minikube docker daemon\neval $(minikube docker-env -u)\n"))),Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Deploy new pods in the cluster"),Object(b.b)("pre",{parentName:"li"},Object(b.b)("code",{parentName:"pre",className:"language-bash"},"helm upgrade --set submarine.database.dev=true submarine ./helm-charts/submarine\n")),Object(b.b)("h2",{parentName:"li",id:"develop-operator"},"Develop operator"))),Object(b.b)("ul",null,Object(b.b)("li",{parentName:"ul"},Object(b.b)("p",{parentName:"li"},"Before building"),Object(b.b)("ol",{parentName:"li"},Object(b.b)("li",{parentName:"ol"},"We assume the developer use ",Object(b.b)("strong",{parentName:"li"},"minikube")," as a local kubernetes cluster."),Object(b.b)("li",{parentName:"ol"},"Make sure you have ",Object(b.b)("strong",{parentName:"li"},"NOT")," installed the submarine helm-chart in the cluster.")))),Object(b.b)("ol",null,Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Start the minikube cluster"),Object(b.b)("pre",{parentName:"li"},Object(b.b)("code",{parentName:"pre",className:"language-bash"},"minikube start --vm-driver=docker --kubernetes-version v1.15.11\n"))),Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Install the dependencies"),Object(b.b)("pre",{parentName:"li"},Object(b.b)("code",{parentName:"pre",className:"language-bash"},"cd submarine-cloud-v2/\ngo mod vendor\n"))),Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Run the operator out-of-cluster"),Object(b.b)("pre",{parentName:"li"},Object(b.b)("code",{parentName:"pre",className:"language-bash"},"make\n./submarine-operator\n"))),Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Deploy a Submarine"),Object(b.b)("pre",{parentName:"li"},Object(b.b)("code",{parentName:"pre",className:"language-bash"},"kubectl apply -f artifacts/examples/crd.yaml\nkubectl create ns submarine-user-test\nkubectl apply -n submarine-user-test -f artifacts/examples/example-submarine.yaml\n"))),Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Exposing service"),Object(b.b)("pre",{parentName:"li"},Object(b.b)("code",{parentName:"pre",className:"language-bash"},"# Method1 -- use minikube ip\nminikube ip  # you'll get the IP address of minikube, ex: 192.168.49.2\n\n# Method2 -- use port-forwarding\nkubectl port-forward --address 0.0.0.0 -n submarine-user-test service/traefik 32080:80\n"))),Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"View workbench"),Object(b.b)("p",{parentName:"li"},"If you use method 1 in step 5, please go to ",Object(b.b)("inlineCode",{parentName:"p"},"http://{minikube ip}:32080"),", ex: ",Object(b.b)("a",{parentName:"p",href:"http://192.168.49.2:32080"},"http://192.168.49.2:32080")),Object(b.b)("p",{parentName:"li"},"If you use method 2 in step 5, please go to ",Object(b.b)("a",{parentName:"p",href:"http://127.0.0.1:32080"},"http://127.0.0.1:32080"))),Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Delete submarine"),Object(b.b)("pre",{parentName:"li"},Object(b.b)("code",{parentName:"pre",className:"language-bash"},"kubectl delete submarine example-submarine -n submarine-user-test\n"))),Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Stop the operator"),Object(b.b)("p",{parentName:"li"},"Press ctrl+c to stop the operator."))),Object(b.b)("p",null,"For other details, please check out the ",Object(b.b)("a",{parentName:"p",href:"https://github.com/apache/submarine/blob/master/submarine-cloud-v2/README.md"},"README")," and ",Object(b.b)("a",{parentName:"p",href:"https://github.com/apache/submarine/blob/master/submarine-cloud-v2/docs/developer-guide.md"},"Developer Guide")," on GitHub."),Object(b.b)("h2",{id:"develop-submarine-website"},"Develop Submarine Website"),Object(b.b)("p",null,"Submarine website is built using ",Object(b.b)("a",{parentName:"p",href:"https://v2.docusaurus.io/"},"Docusaurus 2"),", a modern static website generator."),Object(b.b)("p",null,"We store all the website content in markdown format in the ",Object(b.b)("inlineCode",{parentName:"p"},"submarine/website/docs"),". When committing a new patch to the ",Object(b.b)("inlineCode",{parentName:"p"},"submarine")," repo, Docusaurus will help us generate the ",Object(b.b)("inlineCode",{parentName:"p"},"html")," and ",Object(b.b)("inlineCode",{parentName:"p"},"javascript")," files and push them to  ",Object(b.b)("a",{parentName:"p",href:"https://github.com/apache/submarine-site/tree/asf-site"},"https://github.com/apache/submarine-site/tree/asf-site"),"."),Object(b.b)("p",null,"To update the website, click \u201cEdit this page\u201d on the website."),Object(b.b)("p",null,Object(b.b)("img",{parentName:"p",src:"https://lh4.googleusercontent.com/gYcKpxbsGAKv2giTRqkxOehPGnuvnhE31WjsAsYhFmACIZF3Wh2ipar7mZ7F_KRwecM-L1J8YJAgNigJsJUjqc-5IXeO2XGxCIcYpP9CdSc3YByuUkjT_Bezby2HHtkBLyE1ZY_F",alt:null})),Object(b.b)("h3",{id:"add-a-new-page"},"Add a new page"),Object(b.b)("p",null,"If you want to add a new page to the website, make sure to add the file path to ",Object(b.b)("a",{parentName:"p",href:"https://github.com/apache/submarine/blob/master/website/sidebars.js"},"sidebars.js"),". "),Object(b.b)("h3",{id:"installation"},"Installation"),Object(b.b)("p",null,"We use the yarn package manager to install all dependencies for the website"),Object(b.b)("pre",null,Object(b.b)("code",{parentName:"pre",className:"language-console"},"yarn install\n")),Object(b.b)("h3",{id:"build"},"Build"),Object(b.b)("p",null,"Make sure you can successfully build the website before creating a pull request."),Object(b.b)("pre",null,Object(b.b)("code",{parentName:"pre",className:"language-console"},"yarn build\n")),Object(b.b)("h3",{id:"local-development"},"Local Development"),Object(b.b)("p",null,"This command starts a local development server and open up a browser window. Most changes are reflected live without having to restart the server."),Object(b.b)("pre",null,Object(b.b)("code",{parentName:"pre",className:"language-console"},"yarn start\n")))}s.isMDXComponent=!0},144:function(e,t,a){"use strict";a.d(t,"a",(function(){return p})),a.d(t,"b",(function(){return d}));var n=a(0),r=a.n(n);function b(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function l(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function i(e){for(var t=1;t<arguments.length;t++){var a=null!=arguments[t]?arguments[t]:{};t%2?l(Object(a),!0).forEach((function(t){b(e,t,a[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(a)):l(Object(a)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(a,t))}))}return e}function o(e,t){if(null==e)return{};var a,n,r=function(e,t){if(null==e)return{};var a,n,r={},b=Object.keys(e);for(n=0;n<b.length;n++)a=b[n],t.indexOf(a)>=0||(r[a]=e[a]);return r}(e,t);if(Object.getOwnPropertySymbols){var b=Object.getOwnPropertySymbols(e);for(n=0;n<b.length;n++)a=b[n],t.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(r[a]=e[a])}return r}var c=r.a.createContext({}),s=function(e){var t=r.a.useContext(c),a=t;return e&&(a="function"==typeof e?e(t):i(i({},t),e)),a},p=function(e){var t=s(e.components);return r.a.createElement(c.Provider,{value:t},e.children)},m={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},u=r.a.forwardRef((function(e,t){var a=e.components,n=e.mdxType,b=e.originalType,l=e.parentName,c=o(e,["components","mdxType","originalType","parentName"]),p=s(a),u=n,d=p["".concat(l,".").concat(u)]||p[u]||m[u]||b;return a?r.a.createElement(d,i(i({ref:t},c),{},{components:a})):r.a.createElement(d,i({ref:t},c))}));function d(e,t){var a=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var b=a.length,l=new Array(b);l[0]=u;var i={};for(var o in t)hasOwnProperty.call(t,o)&&(i[o]=t[o]);i.originalType=e,i.mdxType="string"==typeof e?e:n,l[1]=i;for(var c=2;c<b;c++)l[c]=a[c];return r.a.createElement.apply(null,l)}return r.a.createElement.apply(null,a)}u.displayName="MDXCreateElement"}}]);