| (window.webpackJsonp=window.webpackJsonp||[]).push([[63],{130:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return l})),n.d(t,"metadata",(function(){return o})),n.d(t,"toc",(function(){return b})),n.d(t,"default",(function(){return s}));var a=n(3),r=n(7),i=(n(0),n(193)),l={title:"Development Guide"},o={unversionedId:"devDocs/Development",id:"devDocs/Development",isDocsHomePage:!1,title:"Development Guide",description:"\x3c!--",source:"@site/docs/devDocs/Development.md",slug:"/devDocs/Development",permalink:"/docs/next/devDocs/Development",editUrl:"https://github.com/apache/submarine/edit/master/website/docs/devDocs/Development.md",version:"current",sidebar:"docs",previous:{title:"How to Build Submarine",permalink:"/docs/next/devDocs/BuildFromCode"},next:{title:"How to Run Integration K8s Test",permalink:"/docs/next/devDocs/IntegrationTestK8s"}},b=[{value:"Video",id:"video",children:[]},{value:"Develop server",id:"develop-server",children:[{value:"Prerequisites",id:"prerequisites",children:[]},{value:"Setting up checkstyle in IDE",id:"setting-up-checkstyle-in-ide",children:[]},{value:"Testing",id:"testing",children:[]},{value:"Build from source",id:"build-from-source",children:[]}]},{value:"Develop workbench",id:"develop-workbench",children:[]},{value:"Develop database",id:"develop-database",children:[]},{value:"Develop operator",id:"develop-operator",children:[]},{value:"Develop Submarine Website",id:"develop-submarine-website",children:[{value:"Add a new page",id:"add-a-new-page",children:[]},{value:"Installation",id:"installation",children:[]},{value:"Build",id:"build",children:[]},{value:"Local Development",id:"local-development",children:[]}]}],c={toc:b};function s(e){var t=e.components,n=Object(r.a)(e,["components"]);return Object(i.b)("wrapper",Object(a.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h1",{id:"project-overview"},"Project Overview"),Object(i.b)("p",null,"The document ",Object(i.b)("a",{parentName:"p",href:"/docs/next/gettingStarted/quickstart"},"Submarine Quickstart")," shows how to deploy the Submarine service to your Kubernetes cluster. The Submarine service consists mainly of nine components, and you can check them with the following command:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"kubectl get pods -n ${your_namespace}\n")),Object(i.b)("p",null,"A brief introduction about these components:"),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},Object(i.b)("strong",{parentName:"li"},"tf-operator"),": Enable users to run TensorFlow jobs distributedly"),Object(i.b)("li",{parentName:"ol"},Object(i.b)("strong",{parentName:"li"},"pytorch-operator"),": Enable users to run PyTorch jobs distributedly"),Object(i.b)("li",{parentName:"ol"},Object(i.b)("strong",{parentName:"li"},"notebook-controller"),": Jupyter Notebook controller"),Object(i.b)("li",{parentName:"ol"},Object(i.b)("strong",{parentName:"li"},"submarine-traefik"),": Kubernetes Ingress controller"),Object(i.b)("li",{parentName:"ol"},Object(i.b)("strong",{parentName:"li"},"submarine-database"),": A MySQL database to store metadata"),Object(i.b)("li",{parentName:"ol"},Object(i.b)("strong",{parentName:"li"},"submarine-minio"),": An object store for machine learning artifacts"),Object(i.b)("li",{parentName:"ol"},Object(i.b)("strong",{parentName:"li"},"submarine-mlflow"),": A platform for model management"),Object(i.b)("li",{parentName:"ol"},Object(i.b)("strong",{parentName:"li"},"submarine-tensorboard"),": A visualization tool for distributed training experiments"),Object(i.b)("li",{parentName:"ol"},Object(i.b)("strong",{parentName:"li"},"submarine-server"),": Handle API requests, and submit distributed training experiments to Kubernetes.")),Object(i.b)("h1",{id:"submarine-development"},"Submarine Development"),Object(i.b)("h2",{id:"video"},"Video"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"From this ",Object(i.b)("a",{parentName:"li",href:"https://youtu.be/32Na2k6Alv4"},"Video"),", you will know how to deal with the configuration of Submarine and be able to contribute to it via Github.")),Object(i.b)("h2",{id:"develop-server"},"Develop server"),Object(i.b)("h3",{id:"prerequisites"},"Prerequisites"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"JDK 1.8"),Object(i.b)("li",{parentName:"ul"},"Maven 3.3 or later ( < 3.8.1 )"),Object(i.b)("li",{parentName:"ul"},"Docker")),Object(i.b)("h3",{id:"setting-up-checkstyle-in-ide"},"Setting up checkstyle in IDE"),Object(i.b)("p",null,"Checkstyle plugin may help to detect violations directly from the IDE."),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"Install Checkstyle+IDEA plugin from ",Object(i.b)("inlineCode",{parentName:"li"},"Preference")," -> ",Object(i.b)("inlineCode",{parentName:"li"},"Plugins")),Object(i.b)("li",{parentName:"ol"},"Open ",Object(i.b)("inlineCode",{parentName:"li"},"Preference")," -> ",Object(i.b)("inlineCode",{parentName:"li"},"Tools")," -> ",Object(i.b)("inlineCode",{parentName:"li"},"Checkstyle"),Object(i.b)("ol",{parentName:"li"},Object(i.b)("li",{parentName:"ol"},"Set Checkstyle version:",Object(i.b)("ul",{parentName:"li"},Object(i.b)("li",{parentName:"ul"},"Checkstyle version: 8.0"))),Object(i.b)("li",{parentName:"ol"},"Add (+) a new Configuration File",Object(i.b)("ul",{parentName:"li"},Object(i.b)("li",{parentName:"ul"},"Description: Submarine"),Object(i.b)("li",{parentName:"ul"},"Use a local checkstyle ",Object(i.b)("inlineCode",{parentName:"li"},"${SUBMARINE_HOME}/dev-support/maven-config/checkstyle.xml")))))),Object(i.b)("li",{parentName:"ol"},"Open the Checkstyle Tool Window, select the Submarine rule and execute the check")),Object(i.b)("h3",{id:"testing"},"Testing"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},Object(i.b)("p",{parentName:"li"},"Unit Test"),Object(i.b)("p",{parentName:"li"},"For each class, there is a corresponding testClass. For example, ",Object(i.b)("inlineCode",{parentName:"p"},"SubmarineServerTest")," is used for testing ",Object(i.b)("inlineCode",{parentName:"p"},"SubmarineServer"),". Whenever you add a funtion in classes, you must write a unit test to test it.")),Object(i.b)("li",{parentName:"ul"},Object(i.b)("p",{parentName:"li"},"Integration Test: ",Object(i.b)("a",{parentName:"p",href:"/docs/next/devDocs/IntegrationTestK8s"},"IntegrationTestK8s.md")))),Object(i.b)("h3",{id:"build-from-source"},"Build from source"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},Object(i.b)("p",{parentName:"li"},"Before building"),Object(i.b)("ol",{parentName:"li"},Object(i.b)("li",{parentName:"ol"},"We assume the developer use ",Object(i.b)("strong",{parentName:"li"},"minikube")," as a local kubernetes cluster."),Object(i.b)("li",{parentName:"ol"},"Make sure you have ",Object(i.b)("strong",{parentName:"li"},"installed the submarine helm-chart")," in the cluster.")))),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Package the Submarine server into a new jar file"),Object(i.b)("pre",{parentName:"li"},Object(i.b)("code",{parentName:"pre",className:"language-bash"},"mvn install -DskipTests\n"))),Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Build the new server docker image in minikube"),Object(i.b)("pre",{parentName:"li"},Object(i.b)("code",{parentName:"pre",className:"language-bash"},"# switch to minikube docker daemon to build image directly in minikube\neval $(minikube docker-env)\n\n# run docker build\n./dev-support/docker-images/submarine/build.sh\n\n# exit minikube docker daemon\neval $(minikube docker-env -u)\n"))),Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Delete the server deployment and the operator will create a new one using the new image"),Object(i.b)("pre",{parentName:"li"},Object(i.b)("code",{parentName:"pre",className:"language-bash"},"kubectl delete deployment submarine-server\n")))),Object(i.b)("h2",{id:"develop-workbench"},"Develop workbench"),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Deploy the Submarine"),Object(i.b)("p",{parentName:"li"},"Follow ",Object(i.b)("a",{parentName:"p",href:"/docs/next/gettingStarted/quickstart"},"Getting Started/Quickstart"),", and make sure you can connect to ",Object(i.b)("inlineCode",{parentName:"p"},"http://localhost:32080")," in the browser.")),Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Install the dependencies"),Object(i.b)("pre",{parentName:"li"},Object(i.b)("code",{parentName:"pre",className:"language-bash"},"cd submarine-workbench/workbench-web\nnpm install\n"))),Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Run the workbench based on proxy server"),Object(i.b)("pre",{parentName:"li"},Object(i.b)("code",{parentName:"pre",className:"language-bash"},"npm run start\n")),Object(i.b)("ol",{parentName:"li"},Object(i.b)("li",{parentName:"ol"},"The request sent to ",Object(i.b)("inlineCode",{parentName:"li"},"http://localhost:4200")," will be redirected to ",Object(i.b)("inlineCode",{parentName:"li"},"http://localhost:32080"),"."),Object(i.b)("li",{parentName:"ol"},"Open ",Object(i.b)("inlineCode",{parentName:"li"},"http://localhost:4200")," in browser to see the real-time change of workbench."))),Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Frontend E2E test: ",Object(i.b)("a",{parentName:"p",href:"/docs/next/devDocs/IntegrationTestE2E"},"IntegrationTestE2E.md")))),Object(i.b)("h2",{id:"develop-database"},"Develop database"),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Build the docker image"),Object(i.b)("pre",{parentName:"li"},Object(i.b)("code",{parentName:"pre",className:"language-bash"},"# switch to minikube docker daemon to build image directly in minikube\neval $(minikube docker-env)\n\n# run docker build\n./dev-support/docker-images/database/build.sh\n\n# exit minikube docker daemon\neval $(minikube docker-env -u)\n"))),Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Deploy new pods in the cluster"),Object(i.b)("pre",{parentName:"li"},Object(i.b)("code",{parentName:"pre",className:"language-bash"},"helm upgrade --set submarine.database.dev=true submarine ./helm-charts/submarine\n")))),Object(i.b)("h2",{id:"develop-operator"},"Develop operator"),Object(i.b)("p",null,"For details, please check out the ",Object(i.b)("a",{parentName:"p",href:"https://github.com/apache/submarine/blob/master/submarine-cloud-v2/README.md"},"README")," and ",Object(i.b)("a",{parentName:"p",href:"https://github.com/apache/submarine/blob/master/submarine-cloud-v2/docs/developer-guide.md"},"Developer Guide")," on GitHub."),Object(i.b)("h2",{id:"develop-submarine-website"},"Develop Submarine Website"),Object(i.b)("p",null,"Submarine website is built using ",Object(i.b)("a",{parentName:"p",href:"https://v2.docusaurus.io/"},"Docusaurus 2"),", a modern static website generator."),Object(i.b)("p",null,"We store all the website content in markdown format in the ",Object(i.b)("inlineCode",{parentName:"p"},"submarine/website/docs"),". When committing a new patch to the ",Object(i.b)("inlineCode",{parentName:"p"},"submarine")," repo, Docusaurus will help us generate the ",Object(i.b)("inlineCode",{parentName:"p"},"html")," and ",Object(i.b)("inlineCode",{parentName:"p"},"javascript")," files and push them to ",Object(i.b)("a",{parentName:"p",href:"https://github.com/apache/submarine-site/tree/asf-site"},"https://github.com/apache/submarine-site/tree/asf-site"),"."),Object(i.b)("p",null,"To update the website, click \u201cEdit this page\u201d on the website."),Object(i.b)("p",null,Object(i.b)("img",{parentName:"p",src:"https://lh4.googleusercontent.com/gYcKpxbsGAKv2giTRqkxOehPGnuvnhE31WjsAsYhFmACIZF3Wh2ipar7mZ7F_KRwecM-L1J8YJAgNigJsJUjqc-5IXeO2XGxCIcYpP9CdSc3YByuUkjT_Bezby2HHtkBLyE1ZY_F",alt:null})),Object(i.b)("h3",{id:"add-a-new-page"},"Add a new page"),Object(i.b)("p",null,"If you want to add a new page to the website, make sure to add the file path to ",Object(i.b)("a",{parentName:"p",href:"https://github.com/apache/submarine/blob/master/website/sidebars.js"},"sidebars.js"),"."),Object(i.b)("h3",{id:"installation"},"Installation"),Object(i.b)("p",null,"We use the yarn package manager to install all dependencies for the website"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-console"},"yarn install\n")),Object(i.b)("h3",{id:"build"},"Build"),Object(i.b)("p",null,"Make sure you can successfully build the website before creating a pull request."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-console"},"yarn build\n")),Object(i.b)("h3",{id:"local-development"},"Local Development"),Object(i.b)("p",null,"This command starts a local development server and open up a browser window. Most changes are reflected live without having to restart the server."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-console"},"yarn start\n")))}s.isMDXComponent=!0},193:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return d}));var a=n(0),r=n.n(a);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function o(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function b(e,t){if(null==e)return{};var n,a,r=function(e,t){if(null==e)return{};var n,a,r={},i=Object.keys(e);for(a=0;a<i.length;a++)n=i[a],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(a=0;a<i.length;a++)n=i[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var c=r.a.createContext({}),s=function(e){var t=r.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):o(o({},t),e)),n},p=function(e){var t=s(e.components);return r.a.createElement(c.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},m=r.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,i=e.originalType,l=e.parentName,c=b(e,["components","mdxType","originalType","parentName"]),p=s(n),m=a,d=p["".concat(l,".").concat(m)]||p[m]||u[m]||i;return n?r.a.createElement(d,o(o({ref:t},c),{},{components:n})):r.a.createElement(d,o({ref:t},c))}));function d(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var i=n.length,l=new Array(i);l[0]=m;var o={};for(var b in t)hasOwnProperty.call(t,b)&&(o[b]=t[b]);o.originalType=e,o.mdxType="string"==typeof e?e:a,l[1]=o;for(var c=2;c<i;c++)l[c]=n[c];return r.a.createElement.apply(null,l)}return r.a.createElement.apply(null,n)}m.displayName="MDXCreateElement"}}]); |