| (window.webpackJsonp=window.webpackJsonp||[]).push([[33],{100:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return o})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return b})),n.d(t,"default",(function(){return p}));var a=n(3),r=n(7),i=(n(0),n(136)),o={title:"Development Guide"},l={unversionedId:"devDocs/Development",id:"devDocs/Development",isDocsHomePage:!1,title:"Development Guide",description:"\x3c!--",source:"@site/docs/devDocs/Development.md",slug:"/devDocs/Development",permalink:"/docs/devDocs/Development",editUrl:"https://github.com/apache/submarine/edit/master/website/docs/devDocs/Development.md",version:"current",sidebar:"docs",previous:{title:"How to Build Submarine",permalink:"/docs/devDocs/BuildFromCode"},next:{title:"How to Run Integration Test",permalink:"/docs/devDocs/IntegrationTest"}},b=[{value:"Overview",id:"overview",children:[]},{value:"Video",id:"video",children:[]},{value:"Develop server",id:"develop-server",children:[{value:"Prerequisites",id:"prerequisites",children:[]},{value:"Setting up checkstyle in IDE",id:"setting-up-checkstyle-in-ide",children:[]},{value:"Testing",id:"testing",children:[]},{value:"Build from source",id:"build-from-source",children:[]}]},{value:"Develop workbench",id:"develop-workbench",children:[]},{value:"Develop database",id:"develop-database",children:[]}],c={toc:b};function p(e){var t=e.components,n=Object(r.a)(e,["components"]);return Object(i.b)("wrapper",Object(a.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"overview"},"Overview"),Object(i.b)("p",null,"From ",Object(i.b)("a",{parentName:"p",href:"/docs/"},"Getting Started/Submarine Local Deployment"),", you already know that Submarine is installed and uninstalled by Helm. As you can see by ",Object(i.b)("inlineCode",{parentName:"p"},"kubectl get pods"),", there are six major components in Submarine, including ",Object(i.b)("inlineCode",{parentName:"p"},"notebook-controller"),", ",Object(i.b)("inlineCode",{parentName:"p"},"pytorch-operator"),", ",Object(i.b)("inlineCode",{parentName:"p"},"submarine-database"),", ",Object(i.b)("inlineCode",{parentName:"p"},"submarine-server"),", ",Object(i.b)("inlineCode",{parentName:"p"},"submarine-traefik")," and ",Object(i.b)("inlineCode",{parentName:"p"},"tf-job-operator"),". They are launched as pods in kubernetes from the corresponding docker images."),Object(i.b)("p",null,"Some of the components are borrowed from other projects (kubeflow, traefik), including ",Object(i.b)("inlineCode",{parentName:"p"},"notebook-controller"),", ",Object(i.b)("inlineCode",{parentName:"p"},"pytorch-operator"),", ",Object(i.b)("inlineCode",{parentName:"p"},"submarine-traefik")," and ",Object(i.b)("inlineCode",{parentName:"p"},"tf-job-operator"),". The rest of them are built by ourselves, including ",Object(i.b)("inlineCode",{parentName:"p"},"submarine-database")," and ",Object(i.b)("inlineCode",{parentName:"p"},"submarine-server"),"."),Object(i.b)("p",null,"The purpose of the components are as the following:"),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},Object(i.b)("inlineCode",{parentName:"p"},"tf-job-operator"),": manage the operation of tensorflow jobs")),Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},Object(i.b)("inlineCode",{parentName:"p"},"pytorch-operator"),": manage the operation of pytorch jobs")),Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},Object(i.b)("inlineCode",{parentName:"p"},"notebook-controller"),": manage the operation of notebook instances")),Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},Object(i.b)("inlineCode",{parentName:"p"},"submarine-traefik"),": manage the ingress service")),Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},Object(i.b)("inlineCode",{parentName:"p"},"submarine-database"),": store metadata in mysql database")),Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},Object(i.b)("inlineCode",{parentName:"p"},"submarine-server"),": handle api request, submit job to container orchestration, and connect with database."))),Object(i.b)("p",null,"In this document, we only focus on the last two components. You can learn how to develop server, database, and workbench here."),Object(i.b)("h2",{id:"video"},"Video"),Object(i.b)("p",null,"From ",Object(i.b)("a",{parentName:"p",href:"https://youtu.be/32Na2k6Alv4"},"This Video"),", you will know how to deal with the configuration of Submarine and be able to contribute to it via Github."),Object(i.b)("h2",{id:"develop-server"},"Develop server"),Object(i.b)("h3",{id:"prerequisites"},"Prerequisites"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"JDK 1.8"),Object(i.b)("li",{parentName:"ul"},"Maven 3.3 or later ( 3.6.2 is known to fail, see SUBMARINE-273 )"),Object(i.b)("li",{parentName:"ul"},"Docker")),Object(i.b)("h3",{id:"setting-up-checkstyle-in-ide"},"Setting up checkstyle in IDE"),Object(i.b)("p",null,"Checkstyle plugin may help to detect violations directly from the IDE."),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"Install Checkstyle+IDEA plugin from Preference -> Plugins"),Object(i.b)("li",{parentName:"ol"},"Open Preference -> Tools -> Checkstyle ->",Object(i.b)("ol",{parentName:"li"},Object(i.b)("li",{parentName:"ol"},"Set Checkstyle version:",Object(i.b)("ul",{parentName:"li"},Object(i.b)("li",{parentName:"ul"},"Checkstyle version: 8.0"))),Object(i.b)("li",{parentName:"ol"},"Add (+) a new Configuration File",Object(i.b)("ul",{parentName:"li"},Object(i.b)("li",{parentName:"ul"},"Description: Submarine"),Object(i.b)("li",{parentName:"ul"},"Use a local checkstyle ${SUBMARINE_HOME}/dev-support/maven-config/checkstyle.xml"))))),Object(i.b)("li",{parentName:"ol"},"Open the Checkstyle Tool Window, select the Submarine rule and execute the check")),Object(i.b)("h3",{id:"testing"},"Testing"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},Object(i.b)("p",{parentName:"li"},"Unit Test"),Object(i.b)("p",{parentName:"li"},"For each class, there is a corresponding testClass. For example, ",Object(i.b)("inlineCode",{parentName:"p"},"SubmarineServerTest")," is used for testing ",Object(i.b)("inlineCode",{parentName:"p"},"SubmarineServer"),". Whenever you add a funtion in classes, you must write a unit test to test it.")),Object(i.b)("li",{parentName:"ul"},Object(i.b)("p",{parentName:"li"},"Integration Test"),Object(i.b)("p",{parentName:"li"},"See ",Object(i.b)("a",{parentName:"p",href:"/docs/devDocs/IntegrationTest"},"IntegrationTest.md")))),Object(i.b)("h3",{id:"build-from-source"},"Build from source"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},Object(i.b)("p",{parentName:"li"},"Before building"),Object(i.b)("ol",{parentName:"li"},Object(i.b)("li",{parentName:"ol"},"We assume the developer use ",Object(i.b)("strong",{parentName:"li"},"minikube")," as a local kubernetes cluster."),Object(i.b)("li",{parentName:"ol"},"Make sure you have ",Object(i.b)("strong",{parentName:"li"},"installed the submarine helm-chart")," in the cluster.")))),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Package the Submarine server into a new jar file"),Object(i.b)("pre",{parentName:"li"},Object(i.b)("code",{parentName:"pre",className:"language-bash"},"mvn package -DskipTests\n"))),Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Build the new server docker image in minikube"),Object(i.b)("pre",{parentName:"li"},Object(i.b)("code",{parentName:"pre",className:"language-bash"},"# switch to minikube docker daemon to build image directly in minikube\neval $(minikube docker-env)\n\n# run docker build\n./dev-support/docker-images/submarine/build.sh\n\n# exit minikube docker daemon\neval $(minikube docker-env -u)\n"))),Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Update server pod"),Object(i.b)("pre",{parentName:"li"},Object(i.b)("code",{parentName:"pre",className:"language-bash"},"helm upgrade --set submarine.server.dev=true submarine ./helm-charts/submarine\n")),Object(i.b)("p",{parentName:"li"},"Set ",Object(i.b)("inlineCode",{parentName:"p"},"submarine.server.dev")," to ",Object(i.b)("inlineCode",{parentName:"p"},"true"),", enabling the server pod to be launched with the new docker image."))),Object(i.b)("h2",{id:"develop-workbench"},"Develop workbench"),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Deploy the Submarine"),Object(i.b)("p",{parentName:"li"},"Follow ",Object(i.b)("a",{parentName:"p",href:"/docs/"},"Getting Started/Submarine Local Deployment"),", and make sure you can connect to ",Object(i.b)("inlineCode",{parentName:"p"},"http://localhost:32080")," in the browser.")),Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Install the dependencies"),Object(i.b)("pre",{parentName:"li"},Object(i.b)("code",{parentName:"pre",className:"language-bash"},"cd submarine-workbench/workbench-web\nnpm install\n"))),Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Run the workbench based on proxy server"),Object(i.b)("pre",{parentName:"li"},Object(i.b)("code",{parentName:"pre",className:"language-bash"},"npm run start\n")),Object(i.b)("ol",{parentName:"li"},Object(i.b)("li",{parentName:"ol"},"The request sent to ",Object(i.b)("inlineCode",{parentName:"li"},"http://localhost:4200")," will be redirected to ",Object(i.b)("inlineCode",{parentName:"li"},"http://localhost:32080"),"."),Object(i.b)("li",{parentName:"ol"},"Open ",Object(i.b)("inlineCode",{parentName:"li"},"http://localhost:4200")," in browser to see the real-time change of workbench.")))),Object(i.b)("h2",{id:"develop-database"},"Develop database"),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Build the docker image"),Object(i.b)("pre",{parentName:"li"},Object(i.b)("code",{parentName:"pre",className:"language-bash"},"# switch to minikube docker daemon to build image directly in minikube\neval $(minikube docker-env)\n\n# run docker build\n./dev-support/docker-images/database/build.sh\n\n# exit minikube docker daemon\neval $(minikube docker-env -u)\n"))),Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Deploy new pods in the cluster"),Object(i.b)("pre",{parentName:"li"},Object(i.b)("code",{parentName:"pre",className:"language-bash"},"helm upgrade --set submarine.database.dev=true submarine ./helm-charts/submarine\n")))))}p.isMDXComponent=!0},136:function(e,t,n){"use strict";n.d(t,"a",(function(){return s})),n.d(t,"b",(function(){return d}));var a=n(0),r=n.n(a);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?o(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):o(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function b(e,t){if(null==e)return{};var n,a,r=function(e,t){if(null==e)return{};var n,a,r={},i=Object.keys(e);for(a=0;a<i.length;a++)n=i[a],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(a=0;a<i.length;a++)n=i[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var c=r.a.createContext({}),p=function(e){var t=r.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},s=function(e){var t=p(e.components);return r.a.createElement(c.Provider,{value:t},e.children)},m={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},u=r.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,i=e.originalType,o=e.parentName,c=b(e,["components","mdxType","originalType","parentName"]),s=p(n),u=a,d=s["".concat(o,".").concat(u)]||s[u]||m[u]||i;return n?r.a.createElement(d,l(l({ref:t},c),{},{components:n})):r.a.createElement(d,l({ref:t},c))}));function d(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var i=n.length,o=new Array(i);o[0]=u;var l={};for(var b in t)hasOwnProperty.call(t,b)&&(l[b]=t[b]);l.originalType=e,l.mdxType="string"==typeof e?e:a,o[1]=l;for(var c=2;c<i;c++)o[c]=n[c];return r.a.createElement.apply(null,o)}return r.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"}}]); |