| "use strict";(self.webpackChunkwebsite=self.webpackChunkwebsite||[]).push([[238],{3905:function(e,t,a){a.d(t,{Zo:function(){return m},kt:function(){return d}});var n=a(7294);function r(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function l(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function i(e){for(var t=1;t<arguments.length;t++){var a=null!=arguments[t]?arguments[t]:{};t%2?l(Object(a),!0).forEach((function(t){r(e,t,a[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(a)):l(Object(a)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(a,t))}))}return e}function o(e,t){if(null==e)return{};var a,n,r=function(e,t){if(null==e)return{};var a,n,r={},l=Object.keys(e);for(n=0;n<l.length;n++)a=l[n],t.indexOf(a)>=0||(r[a]=e[a]);return r}(e,t);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(n=0;n<l.length;n++)a=l[n],t.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(r[a]=e[a])}return r}var s=n.createContext({}),p=function(e){var t=n.useContext(s),a=t;return e&&(a="function"==typeof e?e(t):i(i({},t),e)),a},m=function(e){var t=p(e.components);return n.createElement(s.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},c=n.forwardRef((function(e,t){var a=e.components,r=e.mdxType,l=e.originalType,s=e.parentName,m=o(e,["components","mdxType","originalType","parentName"]),c=p(a),d=r,k=c["".concat(s,".").concat(d)]||c[d]||u[d]||l;return a?n.createElement(k,i(i({ref:t},m),{},{components:a})):n.createElement(k,i({ref:t},m))}));function d(e,t){var a=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var l=a.length,i=new Array(l);i[0]=c;var o={};for(var s in t)hasOwnProperty.call(t,s)&&(o[s]=t[s]);o.originalType=e,o.mdxType="string"==typeof e?e:r,i[1]=o;for(var p=2;p<l;p++)i[p]=a[p];return n.createElement.apply(null,i)}return n.createElement.apply(null,a)}c.displayName="MDXCreateElement"},9297:function(e,t,a){a.r(t),a.d(t,{assets:function(){return m},contentTitle:function(){return s},default:function(){return d},frontMatter:function(){return o},metadata:function(){return p},toc:function(){return u}});var n=a(7462),r=a(3366),l=(a(7294),a(3905)),i=["components"],o={title:"Development Guide"},s=void 0,p={unversionedId:"devDocs/Development",id:"version-0.6.0/devDocs/Development",title:"Development Guide",description:"\x3c!--",source:"@site/versioned_docs/version-0.6.0/devDocs/Development.md",sourceDirName:"devDocs",slug:"/devDocs/Development",permalink:"/docs/devDocs/Development",editUrl:"https://github.com/apache/submarine/edit/master/website/versioned_docs/version-0.6.0/devDocs/Development.md",tags:[],version:"0.6.0",frontMatter:{title:"Development Guide"},sidebar:"docs",previous:{title:"How to Build Submarine",permalink:"/docs/devDocs/BuildFromCode"},next:{title:"How to Run Integration K8s Test",permalink:"/docs/devDocs/IntegrationTestK8s"}},m={},u=[{value:"Video",id:"video",level:2},{value:"Develop server",id:"develop-server",level:2},{value:"Prerequisites",id:"prerequisites",level:3},{value:"Setting up checkstyle in IDE",id:"setting-up-checkstyle-in-ide",level:3},{value:"Testing",id:"testing",level:3},{value:"Build from source",id:"build-from-source",level:3},{value:"Develop workbench",id:"develop-workbench",level:2},{value:"Develop database",id:"develop-database",level:2},{value:"Develop Submarine Website",id:"develop-submarine-website",level:2},{value:"Add a new page",id:"add-a-new-page",level:3},{value:"Installation",id:"installation",level:3},{value:"Build",id:"build",level:3},{value:"Local Development",id:"local-development",level:3}],c={toc:u};function d(e){var t=e.components,a=(0,r.Z)(e,i);return(0,l.kt)("wrapper",(0,n.Z)({},c,a,{components:t,mdxType:"MDXLayout"}),(0,l.kt)("h1",{id:"project-overview"},"Project Overview"),(0,l.kt)("p",null,"The document ",(0,l.kt)("a",{parentName:"p",href:"/docs/gettingStarted/quickstart"},"Getting Started/Quickstart")," shows how to deploy the Submarine service to your Kubernetes cluster. The Submarine service consists mainly of nine components, and you can check them with the following command:"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre"},"kubectl get pods -n ${your_namespace}\n")),(0,l.kt)("p",null,"A brief introduction about these components: "),(0,l.kt)("ol",null,(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("strong",{parentName:"li"},"tf-operator"),": Enable users to run TensorFlow jobs distributedly"),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("strong",{parentName:"li"},"pytorch-operator"),": Enable users to run PyTorch jobs distributedly"),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("strong",{parentName:"li"},"notebook-controller"),": Jupyter Notebook controller"),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("strong",{parentName:"li"},"submarine-traefik"),": Kubernetes Ingress controller"),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("strong",{parentName:"li"},"submarine-database"),": A MySQL database to store metadata"),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("strong",{parentName:"li"},"submarine-minio"),": An object store for machine learning artifacts"),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("strong",{parentName:"li"},"submarine-mlflow"),": A platform for model management"),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("strong",{parentName:"li"},"submarine-tensorboard"),": A visualization tool for distributed training experiments"),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("strong",{parentName:"li"},"submarine-server"),": Handle API requests, and submit distributed training experiments to Kubernetes.")),(0,l.kt)("h1",{id:"submarine-development"},"Submarine Development"),(0,l.kt)("h2",{id:"video"},"Video"),(0,l.kt)("ul",null,(0,l.kt)("li",{parentName:"ul"},"From this ",(0,l.kt)("a",{parentName:"li",href:"https://youtu.be/32Na2k6Alv4"},"Video"),", you will know how to deal with the configuration of Submarine and be able to contribute to it via Github.")),(0,l.kt)("h2",{id:"develop-server"},"Develop server"),(0,l.kt)("h3",{id:"prerequisites"},"Prerequisites"),(0,l.kt)("ul",null,(0,l.kt)("li",{parentName:"ul"},"JDK 1.8"),(0,l.kt)("li",{parentName:"ul"},"Maven 3.3 or later ( < 3.8.1 )"),(0,l.kt)("li",{parentName:"ul"},"Docker")),(0,l.kt)("h3",{id:"setting-up-checkstyle-in-ide"},"Setting up checkstyle in IDE"),(0,l.kt)("p",null,"Checkstyle plugin may help to detect violations directly from the IDE."),(0,l.kt)("ol",null,(0,l.kt)("li",{parentName:"ol"},"Install Checkstyle+IDEA plugin from Preference -> Plugins"),(0,l.kt)("li",{parentName:"ol"},"Open Preference -> Tools -> Checkstyle ->",(0,l.kt)("ol",{parentName:"li"},(0,l.kt)("li",{parentName:"ol"},"Set Checkstyle version:",(0,l.kt)("ul",{parentName:"li"},(0,l.kt)("li",{parentName:"ul"},"Checkstyle version: 8.0"))),(0,l.kt)("li",{parentName:"ol"},"Add (+) a new Configuration File",(0,l.kt)("ul",{parentName:"li"},(0,l.kt)("li",{parentName:"ul"},"Description: Submarine"),(0,l.kt)("li",{parentName:"ul"},"Use a local checkstyle ${SUBMARINE_HOME}/dev-support/maven-config/checkstyle.xml"))))),(0,l.kt)("li",{parentName:"ol"},"Open the Checkstyle Tool Window, select the Submarine rule and execute the check")),(0,l.kt)("h3",{id:"testing"},"Testing"),(0,l.kt)("ul",null,(0,l.kt)("li",{parentName:"ul"},(0,l.kt)("p",{parentName:"li"},"Unit Test"),(0,l.kt)("p",{parentName:"li"},"For each class, there is a corresponding testClass. For example, ",(0,l.kt)("inlineCode",{parentName:"p"},"SubmarineServerTest")," is used for testing ",(0,l.kt)("inlineCode",{parentName:"p"},"SubmarineServer"),". Whenever you add a funtion in classes, you must write a unit test to test it.")),(0,l.kt)("li",{parentName:"ul"},(0,l.kt)("p",{parentName:"li"},"Integration Test: ",(0,l.kt)("a",{parentName:"p",href:"/docs/devDocs/IntegrationTestK8s"},"IntegrationTestK8s.md")))),(0,l.kt)("h3",{id:"build-from-source"},"Build from source"),(0,l.kt)("ul",null,(0,l.kt)("li",{parentName:"ul"},(0,l.kt)("p",{parentName:"li"},"Before building"),(0,l.kt)("ol",{parentName:"li"},(0,l.kt)("li",{parentName:"ol"},"We assume the developer use ",(0,l.kt)("strong",{parentName:"li"},"minikube")," as a local kubernetes cluster."),(0,l.kt)("li",{parentName:"ol"},"Make sure you have ",(0,l.kt)("strong",{parentName:"li"},"installed the submarine helm-chart")," in the cluster.")))),(0,l.kt)("ol",null,(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("p",{parentName:"li"},"Package the Submarine server into a new jar file"),(0,l.kt)("pre",{parentName:"li"},(0,l.kt)("code",{parentName:"pre",className:"language-bash"},"mvn package -DskipTests\n"))),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("p",{parentName:"li"},"Build the new server docker image in minikube"),(0,l.kt)("pre",{parentName:"li"},(0,l.kt)("code",{parentName:"pre",className:"language-bash"},"# switch to minikube docker daemon to build image directly in minikube\neval $(minikube docker-env)\n\n# run docker build\n./dev-support/docker-images/submarine/build.sh\n\n# exit minikube docker daemon\neval $(minikube docker-env -u)\n"))),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("p",{parentName:"li"},"Update server pod"),(0,l.kt)("pre",{parentName:"li"},(0,l.kt)("code",{parentName:"pre",className:"language-bash"},"helm upgrade --set submarine.server.dev=true submarine ./helm-charts/submarine\n")),(0,l.kt)("p",{parentName:"li"},"Set ",(0,l.kt)("inlineCode",{parentName:"p"},"submarine.server.dev")," to ",(0,l.kt)("inlineCode",{parentName:"p"},"true"),", enabling the server pod to be launched with the new docker image."))),(0,l.kt)("h2",{id:"develop-workbench"},"Develop workbench"),(0,l.kt)("ol",null,(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("p",{parentName:"li"},"Deploy the Submarine"),(0,l.kt)("p",{parentName:"li"},"Follow ",(0,l.kt)("a",{parentName:"p",href:"/docs/gettingStarted/quickstart"},"Getting Started/Quickstart"),", and make sure you can connect to ",(0,l.kt)("inlineCode",{parentName:"p"},"http://localhost:32080")," in the browser.")),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("p",{parentName:"li"},"Install the dependencies"),(0,l.kt)("pre",{parentName:"li"},(0,l.kt)("code",{parentName:"pre",className:"language-bash"},"cd submarine-workbench/workbench-web\nnpm install\n"))),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("p",{parentName:"li"},"Run the workbench based on proxy server"),(0,l.kt)("pre",{parentName:"li"},(0,l.kt)("code",{parentName:"pre",className:"language-bash"},"npm run start\n")),(0,l.kt)("ol",{parentName:"li"},(0,l.kt)("li",{parentName:"ol"},"The request sent to ",(0,l.kt)("inlineCode",{parentName:"li"},"http://localhost:4200")," will be redirected to ",(0,l.kt)("inlineCode",{parentName:"li"},"http://localhost:32080"),"."),(0,l.kt)("li",{parentName:"ol"},"Open ",(0,l.kt)("inlineCode",{parentName:"li"},"http://localhost:4200")," in browser to see the real-time change of workbench."))),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("p",{parentName:"li"},"Frontend E2E test: ",(0,l.kt)("a",{parentName:"p",href:"/docs/devDocs/IntegrationTestE2E"},"IntegrationTestE2E.md")," "))),(0,l.kt)("h2",{id:"develop-database"},"Develop database"),(0,l.kt)("ol",null,(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("p",{parentName:"li"},"Build the docker image"),(0,l.kt)("pre",{parentName:"li"},(0,l.kt)("code",{parentName:"pre",className:"language-bash"},"# switch to minikube docker daemon to build image directly in minikube\neval $(minikube docker-env)\n\n# run docker build\n./dev-support/docker-images/database/build.sh\n\n# exit minikube docker daemon\neval $(minikube docker-env -u)\n"))),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("p",{parentName:"li"},"Deploy new pods in the cluster"),(0,l.kt)("pre",{parentName:"li"},(0,l.kt)("code",{parentName:"pre",className:"language-bash"},"helm upgrade --set submarine.database.dev=true submarine ./helm-charts/submarine\n")),(0,l.kt)("h2",{parentName:"li",id:"develop-operator"},"Develop operator"))),(0,l.kt)("ul",null,(0,l.kt)("li",{parentName:"ul"},(0,l.kt)("p",{parentName:"li"},"Before building"),(0,l.kt)("ol",{parentName:"li"},(0,l.kt)("li",{parentName:"ol"},"We assume the developer use ",(0,l.kt)("strong",{parentName:"li"},"minikube")," as a local kubernetes cluster."),(0,l.kt)("li",{parentName:"ol"},"Make sure you have ",(0,l.kt)("strong",{parentName:"li"},"NOT")," installed the submarine helm-chart in the cluster.")))),(0,l.kt)("ol",null,(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("p",{parentName:"li"},"Start the minikube cluster"),(0,l.kt)("pre",{parentName:"li"},(0,l.kt)("code",{parentName:"pre",className:"language-bash"},"minikube start --vm-driver=docker --kubernetes-version v1.15.11\n"))),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("p",{parentName:"li"},"Install the dependencies"),(0,l.kt)("pre",{parentName:"li"},(0,l.kt)("code",{parentName:"pre",className:"language-bash"},"cd submarine-cloud-v2/\ncp -r ../helm-charts/submarine/charts ./helm-charts/submarine-operator/\ngo mod vendor\nhelm install --set dev=true submarine-operator ./helm-charts/submarine-operator/\n"))),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("p",{parentName:"li"},"Run the operator out-of-cluster"),(0,l.kt)("pre",{parentName:"li"},(0,l.kt)("code",{parentName:"pre",className:"language-bash"},"make\n./submarine-operator\n"))),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("p",{parentName:"li"},"Deploy a Submarine"),(0,l.kt)("pre",{parentName:"li"},(0,l.kt)("code",{parentName:"pre",className:"language-bash"},"kubectl apply -f artifacts/examples/crd.yaml\nkubectl create ns submarine-user-test\nkubectl apply -n submarine-user-test -f artifacts/examples/example-submarine.yaml\n"))),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("p",{parentName:"li"},"Exposing service"),(0,l.kt)("pre",{parentName:"li"},(0,l.kt)("code",{parentName:"pre",className:"language-bash"},"# Method1 -- use minikube ip\nminikube ip # you'll get the IP address of minikube, ex: 192.168.49.2\n\n# Method2 -- use port-forwarding\nkubectl port-forward --address 0.0.0.0 -n submarine-user-test service/traefik 32080:80\n"))),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("p",{parentName:"li"},"View workbench"),(0,l.kt)("p",{parentName:"li"},"If you use method 1 in step 5, please go to ",(0,l.kt)("inlineCode",{parentName:"p"},"http://{minikube ip}:32080"),", ex: ",(0,l.kt)("a",{parentName:"p",href:"http://192.168.49.2:32080"},"http://192.168.49.2:32080")),(0,l.kt)("p",{parentName:"li"},"If you use method 2 in step 5, please go to ",(0,l.kt)("a",{parentName:"p",href:"http://127.0.0.1:32080"},"http://127.0.0.1:32080"))),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("p",{parentName:"li"},"Delete submarine"),(0,l.kt)("pre",{parentName:"li"},(0,l.kt)("code",{parentName:"pre",className:"language-bash"},"kubectl delete submarine example-submarine -n submarine-user-test\n"))),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("p",{parentName:"li"},"Stop the operator"),(0,l.kt)("p",{parentName:"li"},"Press ctrl+c to stop the operator.")),(0,l.kt)("li",{parentName:"ol"},(0,l.kt)("p",{parentName:"li"},"Uninstall helm chart dependencies"),(0,l.kt)("pre",{parentName:"li"},(0,l.kt)("code",{parentName:"pre",className:"language-bash"},"helm delete submarine-operator\n")))),(0,l.kt)("p",null,"For other details, please check out the ",(0,l.kt)("a",{parentName:"p",href:"https://github.com/apache/submarine/blob/master/submarine-cloud-v2/README.md"},"README")," and ",(0,l.kt)("a",{parentName:"p",href:"https://github.com/apache/submarine/blob/master/submarine-cloud-v2/docs/developer-guide.md"},"Developer Guide")," on GitHub."),(0,l.kt)("h2",{id:"develop-submarine-website"},"Develop Submarine Website"),(0,l.kt)("p",null,"Submarine website is built using ",(0,l.kt)("a",{parentName:"p",href:"https://v2.docusaurus.io/"},"Docusaurus 2"),", a modern static website generator."),(0,l.kt)("p",null,"We store all the website content in markdown format in the ",(0,l.kt)("inlineCode",{parentName:"p"},"submarine/website/docs"),". When committing a new patch to the ",(0,l.kt)("inlineCode",{parentName:"p"},"submarine")," repo, Docusaurus will help us generate the ",(0,l.kt)("inlineCode",{parentName:"p"},"html")," and ",(0,l.kt)("inlineCode",{parentName:"p"},"javascript")," files and push them to ",(0,l.kt)("a",{parentName:"p",href:"https://github.com/apache/submarine-site/tree/asf-site"},"https://github.com/apache/submarine-site/tree/asf-site"),"."),(0,l.kt)("p",null,"To update the website, click \u201cEdit this page\u201d on the website."),(0,l.kt)("p",null,(0,l.kt)("img",{parentName:"p",src:"https://lh4.googleusercontent.com/gYcKpxbsGAKv2giTRqkxOehPGnuvnhE31WjsAsYhFmACIZF3Wh2ipar7mZ7F_KRwecM-L1J8YJAgNigJsJUjqc-5IXeO2XGxCIcYpP9CdSc3YByuUkjT_Bezby2HHtkBLyE1ZY_F",alt:null})),(0,l.kt)("h3",{id:"add-a-new-page"},"Add a new page"),(0,l.kt)("p",null,"If you want to add a new page to the website, make sure to add the file path to ",(0,l.kt)("a",{parentName:"p",href:"https://github.com/apache/submarine/blob/master/website/sidebars.js"},"sidebars.js"),". "),(0,l.kt)("h3",{id:"installation"},"Installation"),(0,l.kt)("p",null,"We use the yarn package manager to install all dependencies for the website"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-console"},"yarn install\n")),(0,l.kt)("h3",{id:"build"},"Build"),(0,l.kt)("p",null,"Make sure you can successfully build the website before creating a pull request."),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-console"},"yarn build\n")),(0,l.kt)("h3",{id:"local-development"},"Local Development"),(0,l.kt)("p",null,"This command starts a local development server and open up a browser window. Most changes are reflected live without having to restart the server."),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-console"},"yarn start\n")))}d.isMDXComponent=!0}}]); |