"use strict";(self.webpackChunkwebsite=self.webpackChunkwebsite||[]).push([[3008],{3905:function(e,t,n){n.d(t,{Zo:function(){return u},kt:function(){return d}});var a=n(7294);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function i(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function o(e,t){if(null==e)return{};var n,a,r=function(e,t){if(null==e)return{};var n,a,r={},l=Object.keys(e);for(a=0;a<l.length;a++)n=l[a],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(a=0;a<l.length;a++)n=l[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var s=a.createContext({}),p=function(e){var t=a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},u=function(e){var t=p(e.components);return a.createElement(s.Provider,{value:t},e.children)},m={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},c=a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,l=e.originalType,s=e.parentName,u=o(e,["components","mdxType","originalType","parentName"]),c=p(n),d=r,k=c["".concat(s,".").concat(d)]||c[d]||m[d]||l;return n?a.createElement(k,i(i({ref:t},u),{},{components:n})):a.createElement(k,i({ref:t},u))}));function d(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var l=n.length,i=new Array(l);i[0]=c;var o={};for(var s in t)hasOwnProperty.call(t,s)&&(o[s]=t[s]);o.originalType=e,o.mdxType="string"==typeof e?e:r,i[1]=o;for(var p=2;p<l;p++)i[p]=n[p];return a.createElement.apply(null,i)}return a.createElement.apply(null,n)}c.displayName="MDXCreateElement"},4852:function(e,t,n){n.r(t),n.d(t,{assets:function(){return u},contentTitle:function(){return s},default:function(){return d},frontMatter:function(){return o},metadata:function(){return p},toc:function(){return m}});var a=n(7462),r=n(3366),l=(n(7294),n(3905)),i=["components"],o={title:"Development Guide"},s=void 0,p={unversionedId:"devDocs/Development",id:"version-0.7.0/devDocs/Development",title:"Development Guide",description:"\x3c!--",source:"@site/versioned_docs/version-0.7.0/devDocs/Development.md",sourceDirName:"devDocs",slug:"/devDocs/Development",permalink:"/docs/devDocs/Development",editUrl:"https://github.com/apache/submarine/edit/master/website/versioned_docs/version-0.7.0/devDocs/Development.md",tags:[],version:"0.7.0",frontMatter:{title:"Development Guide"},sidebar:"docs",previous:{title:"How to Build Submarine",permalink:"/docs/devDocs/BuildFromCode"},next:{title:"How to Run Integration K8s Test",permalink:"/docs/devDocs/IntegrationTestK8s"}},u={},m=[{value:"Video",id:"video",level:2},{value:"Develop server",id:"develop-server",level:2},{value:"Prerequisites",id:"prerequisites",level:3},{value:"Setting up checkstyle in IDE",id:"setting-up-checkstyle-in-ide",level:3},{value:"Testing",id:"testing",level:3},{value:"Build from source",id:"build-from-source",level:3},{value:"Develop workbench",id:"develop-workbench",level:2},{value:"Develop database",id:"develop-database",level:2},{value:"Develop operator",id:"develop-operator",level:2},{value:"Develop Submarine Website",id:"develop-submarine-website",level:2},{value:"Add a new page",id:"add-a-new-page",level:3},{value:"Installation",id:"installation",level:3},{value:"Build",id:"build",level:3},{value:"Local Development",id:"local-development",level:3}],c={toc:m};function d(e){var t=e.components,n=(0,r.Z)(e,i);return(0,l.kt)("wrapper",(0,a.Z)({},c,n,{components:t,mdxType:"MDXLayout"}),(0,l.kt)("h1",{id:"project-overview"},"Project Overview"),(0,l.kt)("p",null,"The document ",(0,l.kt)("a",{parentName:"p",href:"/docs/gettingStarted/quickstart"},"Submarine Quickstart")," shows how to deploy the Submarine service to your Kubernetes cluster. The Submarine service consists mainly of nine components, and you can check them with the following command:"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre"},"kubectl get pods -n ${your_namespace}\n")),(0,l.kt)("p",null,"A brief introduction about these components:"),(0,l.kt)("ol",null,(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("strong",{parentName:"li"},"tf-operator"),": Enable users to run TensorFlow jobs distributedly"),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("strong",{parentName:"li"},"pytorch-operator"),": Enable users to run PyTorch jobs distributedly"),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("strong",{parentName:"li"},"notebook-controller"),": Jupyter Notebook controller"),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("strong",{parentName:"li"},"submarine-traefik"),": Kubernetes Ingress controller"),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("strong",{parentName:"li"},"submarine-database"),": A MySQL database to store metadata"),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("strong",{parentName:"li"},"submarine-minio"),": An object store for machine learning artifacts"),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("strong",{parentName:"li"},"submarine-mlflow"),": A platform for model management"),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("strong",{parentName:"li"},"submarine-tensorboard"),": A visualization tool for distributed training experiments"),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("strong",{parentName:"li"},"submarine-server"),": Handle API requests, and submit distributed training experiments to Kubernetes.")),(0,l.kt)("h1",{id:"submarine-development"},"Submarine Development"),(0,l.kt)("h2",{id:"video"},"Video"),(0,l.kt)("ul",null,(0,l.kt)("li",{parentName:"ul"},"From this ",(0,l.kt)("a",{parentName:"li",href:"https://youtu.be/32Na2k6Alv4"},"Video"),", you will know how to deal with the configuration of Submarine and be able to contribute to it via Github.")),(0,l.kt)("h2",{id:"develop-server"},"Develop server"),(0,l.kt)("h3",{id:"prerequisites"},"Prerequisites"),(0,l.kt)("ul",null,(0,l.kt)("li",{parentName:"ul"},"JDK 1.8"),(0,l.kt)("li",{parentName:"ul"},"Maven 3.3 or later ( < 3.8.1 )"),(0,l.kt)("li",{parentName:"ul"},"Docker")),(0,l.kt)("h3",{id:"setting-up-checkstyle-in-ide"},"Setting up checkstyle in IDE"),(0,l.kt)("p",null,"Checkstyle plugin may help to detect violations directly from the IDE."),(0,l.kt)("ol",null,(0,l.kt)("li",{parentName:"ol"},"Install Checkstyle+IDEA plugin from ",(0,l.kt)("inlineCode",{parentName:"li"},"Preference")," -> ",(0,l.kt)("inlineCode",{parentName:"li"},"Plugins")),(0,l.kt)("li",{parentName:"ol"},"Open ",(0,l.kt)("inlineCode",{parentName:"li"},"Preference")," -> ",(0,l.kt)("inlineCode",{parentName:"li"},"Tools")," -> ",(0,l.kt)("inlineCode",{parentName:"li"},"Checkstyle"),(0,l.kt)("ol",{parentName:"li"},(0,l.kt)("li",{parentName:"ol"},"Set Checkstyle version:",(0,l.kt)("ul",{parentName:"li"},(0,l.kt)("li",{parentName:"ul"},"Checkstyle version: 8.0"))),(0,l.kt)("li",{parentName:"ol"},"Add (+) a new Configuration File",(0,l.kt)("ul",{parentName:"li"},(0,l.kt)("li",{parentName:"ul"},"Description: Submarine"),(0,l.kt)("li",{parentName:"ul"},"Use a local checkstyle ",(0,l.kt)("inlineCode",{parentName:"li"},"${SUBMARINE_HOME}/dev-support/maven-config/checkstyle.xml")))))),(0,l.kt)("li",{parentName:"ol"},"Open the Checkstyle Tool Window, select the Submarine rule and execute the check")),(0,l.kt)("h3",{id:"testing"},"Testing"),(0,l.kt)("ul",null,(0,l.kt)("li",{parentName:"ul"},(0,l.kt)("p",{parentName:"li"},"Unit Test"),(0,l.kt)("p",{parentName:"li"},"For each class, there is a corresponding testClass. For example, ",(0,l.kt)("inlineCode",{parentName:"p"},"SubmarineServerTest")," is used for testing ",(0,l.kt)("inlineCode",{parentName:"p"},"SubmarineServer"),". Whenever you add a funtion in classes, you must write a unit test to test it.")),(0,l.kt)("li",{parentName:"ul"},(0,l.kt)("p",{parentName:"li"},"Integration Test: ",(0,l.kt)("a",{parentName:"p",href:"/docs/devDocs/IntegrationTestK8s"},"IntegrationTestK8s.md")))),(0,l.kt)("h3",{id:"build-from-source"},"Build from source"),(0,l.kt)("ul",null,(0,l.kt)("li",{parentName:"ul"},(0,l.kt)("p",{parentName:"li"},"Before building"),(0,l.kt)("ol",{parentName:"li"},(0,l.kt)("li",{parentName:"ol"},"We assume the developer use ",(0,l.kt)("strong",{parentName:"li"},"minikube")," as a local kubernetes cluster."),(0,l.kt)("li",{parentName:"ol"},"Make sure you have ",(0,l.kt)("strong",{parentName:"li"},"installed the submarine helm-chart")," in the cluster.")))),(0,l.kt)("ol",null,(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("p",{parentName:"li"},"Package the Submarine server into a new jar file"),(0,l.kt)("pre",{parentName:"li"},(0,l.kt)("code",{parentName:"pre",className:"language-bash"},"mvn install -DskipTests\n"))),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("p",{parentName:"li"},"Build the new server docker image in minikube"),(0,l.kt)("pre",{parentName:"li"},(0,l.kt)("code",{parentName:"pre",className:"language-bash"},"# switch to minikube docker daemon to build image directly in minikube\neval $(minikube docker-env)\n\n# run docker build\n./dev-support/docker-images/submarine/build.sh\n\n# exit minikube docker daemon\neval $(minikube docker-env -u)\n"))),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("p",{parentName:"li"},"Delete the server deployment and the operator will create a new one using the new image"),(0,l.kt)("pre",{parentName:"li"},(0,l.kt)("code",{parentName:"pre",className:"language-bash"},"kubectl delete deployment submarine-server\n")))),(0,l.kt)("h2",{id:"develop-workbench"},"Develop workbench"),(0,l.kt)("ol",null,(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("p",{parentName:"li"},"Deploy the Submarine"),(0,l.kt)("p",{parentName:"li"},"Follow ",(0,l.kt)("a",{parentName:"p",href:"/docs/gettingStarted/quickstart"},"Getting Started/Quickstart"),", and make sure you can connect to ",(0,l.kt)("inlineCode",{parentName:"p"},"http://localhost:32080")," in the browser.")),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("p",{parentName:"li"},"Install the dependencies"),(0,l.kt)("pre",{parentName:"li"},(0,l.kt)("code",{parentName:"pre",className:"language-bash"},"cd submarine-workbench/workbench-web\nnpm install\n"))),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("p",{parentName:"li"},"Run the workbench based on proxy server"),(0,l.kt)("pre",{parentName:"li"},(0,l.kt)("code",{parentName:"pre",className:"language-bash"},"npm run start\n")),(0,l.kt)("ol",{parentName:"li"},(0,l.kt)("li",{parentName:"ol"},"The request sent to ",(0,l.kt)("inlineCode",{parentName:"li"},"http://localhost:4200")," will be redirected to ",(0,l.kt)("inlineCode",{parentName:"li"},"http://localhost:32080"),"."),(0,l.kt)("li",{parentName:"ol"},"Open ",(0,l.kt)("inlineCode",{parentName:"li"},"http://localhost:4200")," in browser to see the real-time change of workbench."))),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("p",{parentName:"li"},"Frontend E2E test: ",(0,l.kt)("a",{parentName:"p",href:"/docs/devDocs/IntegrationTestE2E"},"IntegrationTestE2E.md")))),(0,l.kt)("h2",{id:"develop-database"},"Develop database"),(0,l.kt)("ol",null,(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("p",{parentName:"li"},"Build the docker image"),(0,l.kt)("pre",{parentName:"li"},(0,l.kt)("code",{parentName:"pre",className:"language-bash"},"# switch to minikube docker daemon to build image directly in minikube\neval $(minikube docker-env)\n\n# run docker build\n./dev-support/docker-images/database/build.sh\n\n# exit minikube docker daemon\neval $(minikube docker-env -u)\n"))),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("p",{parentName:"li"},"Deploy new pods in the cluster"),(0,l.kt)("pre",{parentName:"li"},(0,l.kt)("code",{parentName:"pre",className:"language-bash"},"helm upgrade --set submarine.database.dev=true submarine ./helm-charts/submarine\n")))),(0,l.kt)("h2",{id:"develop-operator"},"Develop operator"),(0,l.kt)("p",null,"For details, please check out the ",(0,l.kt)("a",{parentName:"p",href:"https://github.com/apache/submarine/blob/master/submarine-cloud-v2/README.md"},"README")," and ",(0,l.kt)("a",{parentName:"p",href:"https://github.com/apache/submarine/blob/master/submarine-cloud-v2/docs/developer-guide.md"},"Developer Guide")," on GitHub."),(0,l.kt)("h2",{id:"develop-submarine-website"},"Develop Submarine Website"),(0,l.kt)("p",null,"Submarine website is built using ",(0,l.kt)("a",{parentName:"p",href:"https://v2.docusaurus.io/"},"Docusaurus 2"),", a modern static website generator."),(0,l.kt)("p",null,"We store all the website content in markdown format in the ",(0,l.kt)("inlineCode",{parentName:"p"},"submarine/website/docs"),". When committing a new patch to the ",(0,l.kt)("inlineCode",{parentName:"p"},"submarine")," repo, Docusaurus will help us generate the ",(0,l.kt)("inlineCode",{parentName:"p"},"html")," and ",(0,l.kt)("inlineCode",{parentName:"p"},"javascript")," files and push them to  ",(0,l.kt)("a",{parentName:"p",href:"https://github.com/apache/submarine-site/tree/asf-site"},"https://github.com/apache/submarine-site/tree/asf-site"),"."),(0,l.kt)("p",null,"To update the website, click \u201cEdit this page\u201d on the website."),(0,l.kt)("p",null,(0,l.kt)("img",{parentName:"p",src:"https://lh4.googleusercontent.com/gYcKpxbsGAKv2giTRqkxOehPGnuvnhE31WjsAsYhFmACIZF3Wh2ipar7mZ7F_KRwecM-L1J8YJAgNigJsJUjqc-5IXeO2XGxCIcYpP9CdSc3YByuUkjT_Bezby2HHtkBLyE1ZY_F",alt:null})),(0,l.kt)("h3",{id:"add-a-new-page"},"Add a new page"),(0,l.kt)("p",null,"If you want to add a new page to the website, make sure to add the file path to ",(0,l.kt)("a",{parentName:"p",href:"https://github.com/apache/submarine/blob/master/website/sidebars.js"},"sidebars.js"),"."),(0,l.kt)("h3",{id:"installation"},"Installation"),(0,l.kt)("p",null,"We use the yarn package manager to install all dependencies for the website"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-console"},"yarn install\n")),(0,l.kt)("h3",{id:"build"},"Build"),(0,l.kt)("p",null,"Make sure you can successfully build the website before creating a pull request."),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-console"},"yarn build\n")),(0,l.kt)("h3",{id:"local-development"},"Local Development"),(0,l.kt)("p",null,"This command starts a local development server and open up a browser window. Most changes are reflected live without having to restart the server."),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-console"},"yarn start\n")))}d.isMDXComponent=!0}}]);