| (window.webpackJsonp=window.webpackJsonp||[]).push([[98],{165:function(e,t,a){"use strict";a.r(t),a.d(t,"frontMatter",(function(){return l})),a.d(t,"metadata",(function(){return i})),a.d(t,"toc",(function(){return o})),a.d(t,"default",(function(){return s}));var n=a(3),r=a(7),b=(a(0),a(193)),l={title:"Development Guide"},i={unversionedId:"devDocs/Development",id:"version-0.6.0/devDocs/Development",isDocsHomePage:!1,title:"Development Guide",description:"\x3c!--",source:"@site/versioned_docs/version-0.6.0/devDocs/Development.md",slug:"/devDocs/Development",permalink:"/docs/devDocs/Development",editUrl:"https://github.com/apache/submarine/edit/master/website/versioned_docs/version-0.6.0/devDocs/Development.md",version:"0.6.0",sidebar:"docs",previous:{title:"How to Build Submarine",permalink:"/docs/devDocs/BuildFromCode"},next:{title:"How to Run Integration K8s Test",permalink:"/docs/devDocs/IntegrationTestK8s"}},o=[{value:"Video",id:"video",children:[]},{value:"Develop server",id:"develop-server",children:[{value:"Prerequisites",id:"prerequisites",children:[]},{value:"Setting up checkstyle in IDE",id:"setting-up-checkstyle-in-ide",children:[]},{value:"Testing",id:"testing",children:[]},{value:"Build from source",id:"build-from-source",children:[]}]},{value:"Develop workbench",id:"develop-workbench",children:[]},{value:"Develop database",id:"develop-database",children:[]},{value:"Develop Submarine Website",id:"develop-submarine-website",children:[{value:"Add a new page",id:"add-a-new-page",children:[]},{value:"Installation",id:"installation",children:[]},{value:"Build",id:"build",children:[]},{value:"Local Development",id:"local-development",children:[]}]}],c={toc:o};function s(e){var t=e.components,a=Object(r.a)(e,["components"]);return Object(b.b)("wrapper",Object(n.a)({},c,a,{components:t,mdxType:"MDXLayout"}),Object(b.b)("h1",{id:"project-overview"},"Project Overview"),Object(b.b)("p",null,"The document ",Object(b.b)("a",{parentName:"p",href:"/docs/gettingStarted/quickstart"},"Getting Started/Quickstart")," shows how to deploy the Submarine service to your Kubernetes cluster. The Submarine service consists mainly of nine components, and you can check them with the following command:"),Object(b.b)("pre",null,Object(b.b)("code",{parentName:"pre"},"kubectl get pods -n ${your_namespace}\n")),Object(b.b)("p",null,"A brief introduction about these components: "),Object(b.b)("ol",null,Object(b.b)("li",{parentName:"ol"},Object(b.b)("strong",{parentName:"li"},"tf-operator"),": Enable users to run TensorFlow jobs distributedly"),Object(b.b)("li",{parentName:"ol"},Object(b.b)("strong",{parentName:"li"},"pytorch-operator"),": Enable users to run PyTorch jobs distributedly"),Object(b.b)("li",{parentName:"ol"},Object(b.b)("strong",{parentName:"li"},"notebook-controller"),": Jupyter Notebook controller"),Object(b.b)("li",{parentName:"ol"},Object(b.b)("strong",{parentName:"li"},"submarine-traefik"),": Kubernetes Ingress controller"),Object(b.b)("li",{parentName:"ol"},Object(b.b)("strong",{parentName:"li"},"submarine-database"),": A MySQL database to store metadata"),Object(b.b)("li",{parentName:"ol"},Object(b.b)("strong",{parentName:"li"},"submarine-minio"),": An object store for machine learning artifacts"),Object(b.b)("li",{parentName:"ol"},Object(b.b)("strong",{parentName:"li"},"submarine-mlflow"),": A platform for model management"),Object(b.b)("li",{parentName:"ol"},Object(b.b)("strong",{parentName:"li"},"submarine-tensorboard"),": A visualization tool for distributed training experiments"),Object(b.b)("li",{parentName:"ol"},Object(b.b)("strong",{parentName:"li"},"submarine-server"),": Handle API requests, and submit distributed training experiments to Kubernetes.")),Object(b.b)("h1",{id:"submarine-development"},"Submarine Development"),Object(b.b)("h2",{id:"video"},"Video"),Object(b.b)("ul",null,Object(b.b)("li",{parentName:"ul"},"From this ",Object(b.b)("a",{parentName:"li",href:"https://youtu.be/32Na2k6Alv4"},"Video"),", you will know how to deal with the configuration of Submarine and be able to contribute to it via Github.")),Object(b.b)("h2",{id:"develop-server"},"Develop server"),Object(b.b)("h3",{id:"prerequisites"},"Prerequisites"),Object(b.b)("ul",null,Object(b.b)("li",{parentName:"ul"},"JDK 1.8"),Object(b.b)("li",{parentName:"ul"},"Maven 3.3 or later ( < 3.8.1 )"),Object(b.b)("li",{parentName:"ul"},"Docker")),Object(b.b)("h3",{id:"setting-up-checkstyle-in-ide"},"Setting up checkstyle in IDE"),Object(b.b)("p",null,"Checkstyle plugin may help to detect violations directly from the IDE."),Object(b.b)("ol",null,Object(b.b)("li",{parentName:"ol"},"Install Checkstyle+IDEA plugin from Preference -> Plugins"),Object(b.b)("li",{parentName:"ol"},"Open Preference -> Tools -> Checkstyle ->",Object(b.b)("ol",{parentName:"li"},Object(b.b)("li",{parentName:"ol"},"Set Checkstyle version:",Object(b.b)("ul",{parentName:"li"},Object(b.b)("li",{parentName:"ul"},"Checkstyle version: 8.0"))),Object(b.b)("li",{parentName:"ol"},"Add (+) a new Configuration File",Object(b.b)("ul",{parentName:"li"},Object(b.b)("li",{parentName:"ul"},"Description: Submarine"),Object(b.b)("li",{parentName:"ul"},"Use a local checkstyle ${SUBMARINE_HOME}/dev-support/maven-config/checkstyle.xml"))))),Object(b.b)("li",{parentName:"ol"},"Open the Checkstyle Tool Window, select the Submarine rule and execute the check")),Object(b.b)("h3",{id:"testing"},"Testing"),Object(b.b)("ul",null,Object(b.b)("li",{parentName:"ul"},Object(b.b)("p",{parentName:"li"},"Unit Test"),Object(b.b)("p",{parentName:"li"},"For each class, there is a corresponding testClass. For example, ",Object(b.b)("inlineCode",{parentName:"p"},"SubmarineServerTest")," is used for testing ",Object(b.b)("inlineCode",{parentName:"p"},"SubmarineServer"),". Whenever you add a funtion in classes, you must write a unit test to test it.")),Object(b.b)("li",{parentName:"ul"},Object(b.b)("p",{parentName:"li"},"Integration Test: ",Object(b.b)("a",{parentName:"p",href:"/docs/devDocs/IntegrationTestK8s"},"IntegrationTestK8s.md")))),Object(b.b)("h3",{id:"build-from-source"},"Build from source"),Object(b.b)("ul",null,Object(b.b)("li",{parentName:"ul"},Object(b.b)("p",{parentName:"li"},"Before building"),Object(b.b)("ol",{parentName:"li"},Object(b.b)("li",{parentName:"ol"},"We assume the developer use ",Object(b.b)("strong",{parentName:"li"},"minikube")," as a local kubernetes cluster."),Object(b.b)("li",{parentName:"ol"},"Make sure you have ",Object(b.b)("strong",{parentName:"li"},"installed the submarine helm-chart")," in the cluster.")))),Object(b.b)("ol",null,Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Package the Submarine server into a new jar file"),Object(b.b)("pre",{parentName:"li"},Object(b.b)("code",{parentName:"pre",className:"language-bash"},"mvn package -DskipTests\n"))),Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Build the new server docker image in minikube"),Object(b.b)("pre",{parentName:"li"},Object(b.b)("code",{parentName:"pre",className:"language-bash"},"# switch to minikube docker daemon to build image directly in minikube\neval $(minikube docker-env)\n\n# run docker build\n./dev-support/docker-images/submarine/build.sh\n\n# exit minikube docker daemon\neval $(minikube docker-env -u)\n"))),Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Update server pod"),Object(b.b)("pre",{parentName:"li"},Object(b.b)("code",{parentName:"pre",className:"language-bash"},"helm upgrade --set submarine.server.dev=true submarine ./helm-charts/submarine\n")),Object(b.b)("p",{parentName:"li"},"Set ",Object(b.b)("inlineCode",{parentName:"p"},"submarine.server.dev")," to ",Object(b.b)("inlineCode",{parentName:"p"},"true"),", enabling the server pod to be launched with the new docker image."))),Object(b.b)("h2",{id:"develop-workbench"},"Develop workbench"),Object(b.b)("ol",null,Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Deploy the Submarine"),Object(b.b)("p",{parentName:"li"},"Follow ",Object(b.b)("a",{parentName:"p",href:"/docs/gettingStarted/quickstart"},"Getting Started/Quickstart"),", and make sure you can connect to ",Object(b.b)("inlineCode",{parentName:"p"},"http://localhost:32080")," in the browser.")),Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Install the dependencies"),Object(b.b)("pre",{parentName:"li"},Object(b.b)("code",{parentName:"pre",className:"language-bash"},"cd submarine-workbench/workbench-web\nnpm install\n"))),Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Run the workbench based on proxy server"),Object(b.b)("pre",{parentName:"li"},Object(b.b)("code",{parentName:"pre",className:"language-bash"},"npm run start\n")),Object(b.b)("ol",{parentName:"li"},Object(b.b)("li",{parentName:"ol"},"The request sent to ",Object(b.b)("inlineCode",{parentName:"li"},"http://localhost:4200")," will be redirected to ",Object(b.b)("inlineCode",{parentName:"li"},"http://localhost:32080"),"."),Object(b.b)("li",{parentName:"ol"},"Open ",Object(b.b)("inlineCode",{parentName:"li"},"http://localhost:4200")," in browser to see the real-time change of workbench."))),Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Frontend E2E test: ",Object(b.b)("a",{parentName:"p",href:"/docs/devDocs/IntegrationTestE2E"},"IntegrationTestE2E.md")," "))),Object(b.b)("h2",{id:"develop-database"},"Develop database"),Object(b.b)("ol",null,Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Build the docker image"),Object(b.b)("pre",{parentName:"li"},Object(b.b)("code",{parentName:"pre",className:"language-bash"},"# switch to minikube docker daemon to build image directly in minikube\neval $(minikube docker-env)\n\n# run docker build\n./dev-support/docker-images/database/build.sh\n\n# exit minikube docker daemon\neval $(minikube docker-env -u)\n"))),Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Deploy new pods in the cluster"),Object(b.b)("pre",{parentName:"li"},Object(b.b)("code",{parentName:"pre",className:"language-bash"},"helm upgrade --set submarine.database.dev=true submarine ./helm-charts/submarine\n")),Object(b.b)("h2",{parentName:"li",id:"develop-operator"},"Develop operator"))),Object(b.b)("ul",null,Object(b.b)("li",{parentName:"ul"},Object(b.b)("p",{parentName:"li"},"Before building"),Object(b.b)("ol",{parentName:"li"},Object(b.b)("li",{parentName:"ol"},"We assume the developer use ",Object(b.b)("strong",{parentName:"li"},"minikube")," as a local kubernetes cluster."),Object(b.b)("li",{parentName:"ol"},"Make sure you have ",Object(b.b)("strong",{parentName:"li"},"NOT")," installed the submarine helm-chart in the cluster.")))),Object(b.b)("ol",null,Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Start the minikube cluster"),Object(b.b)("pre",{parentName:"li"},Object(b.b)("code",{parentName:"pre",className:"language-bash"},"minikube start --vm-driver=docker --kubernetes-version v1.15.11\n"))),Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Install the dependencies"),Object(b.b)("pre",{parentName:"li"},Object(b.b)("code",{parentName:"pre",className:"language-bash"},"cd submarine-cloud-v2/\ncp -r ../helm-charts/submarine/charts ./helm-charts/submarine-operator/\ngo mod vendor\nhelm install --set dev=true submarine-operator ./helm-charts/submarine-operator/\n"))),Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Run the operator out-of-cluster"),Object(b.b)("pre",{parentName:"li"},Object(b.b)("code",{parentName:"pre",className:"language-bash"},"make\n./submarine-operator\n"))),Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Deploy a Submarine"),Object(b.b)("pre",{parentName:"li"},Object(b.b)("code",{parentName:"pre",className:"language-bash"},"kubectl apply -f artifacts/examples/crd.yaml\nkubectl create ns submarine-user-test\nkubectl apply -n submarine-user-test -f artifacts/examples/example-submarine.yaml\n"))),Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Exposing service"),Object(b.b)("pre",{parentName:"li"},Object(b.b)("code",{parentName:"pre",className:"language-bash"},"# Method1 -- use minikube ip\nminikube ip # you'll get the IP address of minikube, ex: 192.168.49.2\n\n# Method2 -- use port-forwarding\nkubectl port-forward --address 0.0.0.0 -n submarine-user-test service/traefik 32080:80\n"))),Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"View workbench"),Object(b.b)("p",{parentName:"li"},"If you use method 1 in step 5, please go to ",Object(b.b)("inlineCode",{parentName:"p"},"http://{minikube ip}:32080"),", ex: ",Object(b.b)("a",{parentName:"p",href:"http://192.168.49.2:32080"},"http://192.168.49.2:32080")),Object(b.b)("p",{parentName:"li"},"If you use method 2 in step 5, please go to ",Object(b.b)("a",{parentName:"p",href:"http://127.0.0.1:32080"},"http://127.0.0.1:32080"))),Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Delete submarine"),Object(b.b)("pre",{parentName:"li"},Object(b.b)("code",{parentName:"pre",className:"language-bash"},"kubectl delete submarine example-submarine -n submarine-user-test\n"))),Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Stop the operator"),Object(b.b)("p",{parentName:"li"},"Press ctrl+c to stop the operator.")),Object(b.b)("li",{parentName:"ol"},Object(b.b)("p",{parentName:"li"},"Uninstall helm chart dependencies"),Object(b.b)("pre",{parentName:"li"},Object(b.b)("code",{parentName:"pre",className:"language-bash"},"helm delete submarine-operator\n")))),Object(b.b)("p",null,"For other details, please check out the ",Object(b.b)("a",{parentName:"p",href:"https://github.com/apache/submarine/blob/master/submarine-cloud-v2/README.md"},"README")," and ",Object(b.b)("a",{parentName:"p",href:"https://github.com/apache/submarine/blob/master/submarine-cloud-v2/docs/developer-guide.md"},"Developer Guide")," on GitHub."),Object(b.b)("h2",{id:"develop-submarine-website"},"Develop Submarine Website"),Object(b.b)("p",null,"Submarine website is built using ",Object(b.b)("a",{parentName:"p",href:"https://v2.docusaurus.io/"},"Docusaurus 2"),", a modern static website generator."),Object(b.b)("p",null,"We store all the website content in markdown format in the ",Object(b.b)("inlineCode",{parentName:"p"},"submarine/website/docs"),". When committing a new patch to the ",Object(b.b)("inlineCode",{parentName:"p"},"submarine")," repo, Docusaurus will help us generate the ",Object(b.b)("inlineCode",{parentName:"p"},"html")," and ",Object(b.b)("inlineCode",{parentName:"p"},"javascript")," files and push them to ",Object(b.b)("a",{parentName:"p",href:"https://github.com/apache/submarine-site/tree/asf-site"},"https://github.com/apache/submarine-site/tree/asf-site"),"."),Object(b.b)("p",null,"To update the website, click \u201cEdit this page\u201d on the website."),Object(b.b)("p",null,Object(b.b)("img",{parentName:"p",src:"https://lh4.googleusercontent.com/gYcKpxbsGAKv2giTRqkxOehPGnuvnhE31WjsAsYhFmACIZF3Wh2ipar7mZ7F_KRwecM-L1J8YJAgNigJsJUjqc-5IXeO2XGxCIcYpP9CdSc3YByuUkjT_Bezby2HHtkBLyE1ZY_F",alt:null})),Object(b.b)("h3",{id:"add-a-new-page"},"Add a new page"),Object(b.b)("p",null,"If you want to add a new page to the website, make sure to add the file path to ",Object(b.b)("a",{parentName:"p",href:"https://github.com/apache/submarine/blob/master/website/sidebars.js"},"sidebars.js"),". "),Object(b.b)("h3",{id:"installation"},"Installation"),Object(b.b)("p",null,"We use the yarn package manager to install all dependencies for the website"),Object(b.b)("pre",null,Object(b.b)("code",{parentName:"pre",className:"language-console"},"yarn install\n")),Object(b.b)("h3",{id:"build"},"Build"),Object(b.b)("p",null,"Make sure you can successfully build the website before creating a pull request."),Object(b.b)("pre",null,Object(b.b)("code",{parentName:"pre",className:"language-console"},"yarn build\n")),Object(b.b)("h3",{id:"local-development"},"Local Development"),Object(b.b)("p",null,"This command starts a local development server and open up a browser window. Most changes are reflected live without having to restart the server."),Object(b.b)("pre",null,Object(b.b)("code",{parentName:"pre",className:"language-console"},"yarn start\n")))}s.isMDXComponent=!0},193:function(e,t,a){"use strict";a.d(t,"a",(function(){return p})),a.d(t,"b",(function(){return d}));var n=a(0),r=a.n(n);function b(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function l(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function i(e){for(var t=1;t<arguments.length;t++){var a=null!=arguments[t]?arguments[t]:{};t%2?l(Object(a),!0).forEach((function(t){b(e,t,a[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(a)):l(Object(a)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(a,t))}))}return e}function o(e,t){if(null==e)return{};var a,n,r=function(e,t){if(null==e)return{};var a,n,r={},b=Object.keys(e);for(n=0;n<b.length;n++)a=b[n],t.indexOf(a)>=0||(r[a]=e[a]);return r}(e,t);if(Object.getOwnPropertySymbols){var b=Object.getOwnPropertySymbols(e);for(n=0;n<b.length;n++)a=b[n],t.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(r[a]=e[a])}return r}var c=r.a.createContext({}),s=function(e){var t=r.a.useContext(c),a=t;return e&&(a="function"==typeof e?e(t):i(i({},t),e)),a},p=function(e){var t=s(e.components);return r.a.createElement(c.Provider,{value:t},e.children)},m={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},u=r.a.forwardRef((function(e,t){var a=e.components,n=e.mdxType,b=e.originalType,l=e.parentName,c=o(e,["components","mdxType","originalType","parentName"]),p=s(a),u=n,d=p["".concat(l,".").concat(u)]||p[u]||m[u]||b;return a?r.a.createElement(d,i(i({ref:t},c),{},{components:a})):r.a.createElement(d,i({ref:t},c))}));function d(e,t){var a=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var b=a.length,l=new Array(b);l[0]=u;var i={};for(var o in t)hasOwnProperty.call(t,o)&&(i[o]=t[o]);i.originalType=e,i.mdxType="string"==typeof e?e:n,l[1]=i;for(var c=2;c<b;c++)l[c]=a[c];return r.a.createElement.apply(null,l)}return r.a.createElement.apply(null,a)}u.displayName="MDXCreateElement"}}]); |