(window.webpackJsonp=window.webpackJsonp||[]).push([[61],{128:function(e,n,t){"use strict";t.r(n),t.d(n,"frontMatter",(function(){return o})),t.d(n,"metadata",(function(){return b})),t.d(n,"toc",(function(){return c})),t.d(n,"default",(function(){return s}));var r=t(3),a=t(7),i=(t(0),t(136)),o={title:"Project Architecture"},b={unversionedId:"devDocs/README",id:"devDocs/README",isDocsHomePage:!1,title:"Project Architecture",description:"\x3c!--",source:"@site/docs/devDocs/README.md",slug:"/devDocs/README",permalink:"/docs/devDocs/README",editUrl:"https://github.com/apache/submarine/edit/master/website/docs/devDocs/README.md",version:"current",sidebar:"docs",previous:{title:"Running Submarine on YARN",permalink:"/docs/adminDocs/yarn/README"},next:{title:"How to Build Submarine",permalink:"/docs/devDocs/BuildFromCode"}},c=[{value:"1. Introduction",id:"1-introduction",children:[]},{value:"2. Submarine Project Structure",id:"2-submarine-project-structure",children:[{value:"2.1. submarine-client",id:"21-submarine-client",children:[]},{value:"2.2. submarine-cloud",id:"22-submarine-cloud",children:[]},{value:"2.3. submarine-commons",id:"23-submarine-commons",children:[]},{value:"2.4. submarine-dist",id:"24-submarine-dist",children:[]},{value:"2.5. submarine-sdk",id:"25-submarine-sdk",children:[]},{value:"2.6. submarine-security",id:"26-submarine-security",children:[]},{value:"2.7. submarine-server",id:"27-submarine-server",children:[]},{value:"2.8. submarine-test",id:"28-submarine-test",children:[]},{value:"2.9. submarine-workbench",id:"29-submarine-workbench",children:[]},{value:"2.10 dev-support",id:"210-dev-support",children:[]}]},{value:"3. Submarine Workbench Development Guide",id:"3-submarine-workbench-development-guide",children:[]},{value:"3.1 Database",id:"31-database",children:[]},{value:"3.2 Workbench-web",id:"32-workbench-web",children:[]},{value:"3.3 Workbench-server",id:"33-workbench-server",children:[]}],l={toc:c};function s(e){var n=e.components,o=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(r.a)({},l,o,{components:n,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"1-introduction"},"1. Introduction"),Object(i.b)("p",null,"This document mainly describes the structure of each module of the Submarine project, the development and test description of each module."),Object(i.b)("h2",{id:"2-submarine-project-structure"},"2. Submarine Project Structure"),Object(i.b)("h3",{id:"21-submarine-client"},"2.1. submarine-client"),Object(i.b)("p",null,"Provide the CLI interface for submarine user. (Currently only support YARN service)"),Object(i.b)("h3",{id:"22-submarine-cloud"},"2.2. submarine-cloud"),Object(i.b)("p",null,"Define submarine operator. (Work in progress)"),Object(i.b)("h3",{id:"23-submarine-commons"},"2.3. submarine-commons"),Object(i.b)("p",null,"Define utility function used in multiple packages, mainly related to hadoop."),Object(i.b)("h3",{id:"24-submarine-dist"},"2.4. submarine-dist"),Object(i.b)("p",null,"Store the pre-release files."),Object(i.b)("h3",{id:"25-submarine-sdk"},"2.5. submarine-sdk"),Object(i.b)("p",null,"Provide Python SDK for submarine user."),Object(i.b)("h3",{id:"26-submarine-security"},"2.6. submarine-security"),Object(i.b)("p",null,"Provide authorization for Apache Spark to talking to Ranger Admin."),Object(i.b)("h3",{id:"27-submarine-server"},"2.7. submarine-server"),Object(i.b)("p",null,"Include core server, restful api, and k8s/yarn submitter."),Object(i.b)("h3",{id:"28-submarine-test"},"2.8. submarine-test"),Object(i.b)("p",null,"Provide end-to-end and k8s test for submarine."),Object(i.b)("h3",{id:"29-submarine-workbench"},"2.9. submarine-workbench"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"workbench-server: is a Jetty-based web server service. Workbench-server provides RESTful interface and Websocket interface. The RESTful interface provides workbench-web with management capabilities for databases such as project, department, user, and role."),Object(i.b)("li",{parentName:"ul"},"workbench-web: is a web front-end service based on Angular.js framework. With workbench-web users can manage Submarine project, department, user, role through browser. You can also use the notebook to develop machine learning algorithms, model release and other lifecycle management.")),Object(i.b)("h3",{id:"210-dev-support"},"2.10 dev-support"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},Object(i.b)("strong",{parentName:"li"},"mini-submarine"),": by using the docker image provided by Submarine, you can\nexperience all the functions of Submarine in a single docker environment, while\nmini-submarine also provides developers with a development and testing\nenvironment, Avoid the hassle of installing and deploying the runtime\nenvironment."),Object(i.b)("li",{parentName:"ul"},Object(i.b)("strong",{parentName:"li"},"submarine-installer"),": submarine-installer is our submarine runtime\nenvironment installation tool for yarn-3.1+ and above.By using\nsubmarine-installer, it is easy to install and deploy system services such as\n",Object(i.b)("inlineCode",{parentName:"li"},"docker"),", ",Object(i.b)("inlineCode",{parentName:"li"},"nvidia-docker"),", ",Object(i.b)("inlineCode",{parentName:"li"},"nvidia driver"),", ",Object(i.b)("inlineCode",{parentName:"li"},"ETCD"),", ",Object(i.b)("inlineCode",{parentName:"li"},"Calico network")," etc.\nrequired by yarn-3.1+.")),Object(i.b)("h2",{id:"3-submarine-workbench-development-guide"},"3. Submarine Workbench Development Guide"),Object(i.b)("p",null,"Submarine workbench consists of three modules: workbench-server, workbench-web\nand database."),Object(i.b)("p",null,"First, you need to clone the entire Submarine project:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"git clone https://github.com/apache/submarine.git\n")),Object(i.b)("h2",{id:"31-database"},"3.1 Database"),Object(i.b)("p",null,"Submarine selects mysql as the workbench database, and stores the table\nstructure and information of all the data in workbench in mysql. Please browse\nthe project's ",Object(i.b)("a",{parentName:"p",href:"https://github.com/apache/submarine/tree/master/dev-support/database"},"Submarine Database Guide")," documentation and follow\nthe instructions to install a mysql database via docker in your development and\ntest environment."),Object(i.b)("h2",{id:"32-workbench-web"},"3.2 Workbench-web"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},Object(i.b)("strong",{parentName:"li"},"Install dependencies"))),Object(i.b)("p",null,"You only need to execute the following command when you run workbench-web for\nthe first time, so you can follow the depend. environment of node.js in the\nworkbench-web directory."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-bash"},"cd submarine-workbench/workbench-web\nyarn install\n")),Object(i.b)("p",null,"The node.js dependency library will be installed in the node_modules directory. node_modules does not need to be uploaded to the git repository. We have excluded it in the submarine/.gitignore file. You can clean this by manually deleting the directory or executing mvn clean. table of Contents."),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},Object(i.b)("strong",{parentName:"li"},"Compiles and hot-reloads for development"))),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-bash"},"yarn run build\n")),Object(i.b)("p",null,"By executing the above command, ",Object(i.b)("inlineCode",{parentName:"p"},"workbench-web")," will publish the web page to the ",Object(i.b)("inlineCode",{parentName:"p"},"workbench-web/dist")," directory. Later we will also add the feature of ",Object(i.b)("inlineCode",{parentName:"p"},"workbench-web")," to a WAR package, which only requires a ",Object(i.b)("inlineCode",{parentName:"p"},"workbench-web.war")," file. Package can release submarine workbench."),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},Object(i.b)("strong",{parentName:"li"},"Lints and fixes files"))),Object(i.b)("p",null,"When you write the ",Object(i.b)("inlineCode",{parentName:"p"},"Angular, js")," file in ",Object(i.b)("inlineCode",{parentName:"p"},"workbench-web")," through IDEA, because IDEA can't format these files well, you need to execute the following command to format the angular and js files to avoid some warnings during the yarn build."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"yarn run lint\n")),Object(i.b)("p",null,"In fact, you must execute this command when you develop and submit any workbench-web feature. Otherwise, chances are that you won't be able to check the code specification we set up in Travis."),Object(i.b)("h2",{id:"33-workbench-server"},"3.3 Workbench-server"),Object(i.b)("p",null,"Workbench-server has a built-in jetty server service, so you don't need to install any web container service. You can start submarine workbench by launching ",Object(i.b)("inlineCode",{parentName:"p"},"workbench-server")," directly in IDEA."),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},Object(i.b)("strong",{parentName:"li"},"Run / Debug")," : In the IDEA, add a Run/Debug Configuration, ",Object(i.b)("inlineCode",{parentName:"li"},"Main Class:")," select ",Object(i.b)("inlineCode",{parentName:"li"},"org.apache.submarine.server.SubmarineServer"),", ",Object(i.b)("inlineCode",{parentName:"li"},"Use classpath of module:")," select ",Object(i.b)("inlineCode",{parentName:"li"},"submarine-server-core"),".")),Object(i.b)("p",null,Object(i.b)("img",{alt:"img",src:t(210).default})),Object(i.b)("p",null,"So you can pass ",Object(i.b)("a",{parentName:"p",href:"http://127.0.0.1:8080"},"http://127.0.0.1:8080")," debugging or running to submarine-workbench."),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},Object(i.b)("strong",{parentName:"li"},"It should be noted"),":")),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Because ",Object(i.b)("inlineCode",{parentName:"p"},"workbench-web/dist")," is the webapp default directory of the workbench-server Jetty Server, the workbench-server will automatically load the ",Object(i.b)("inlineCode",{parentName:"p"},"workbench-web/dist")," directory after startup.")),Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"The ",Object(i.b)("inlineCode",{parentName:"p"},"workbench-web/dist")," used by ",Object(i.b)("inlineCode",{parentName:"p"},"workbench-server")," is used as the webapp directory, which is configured via ",Object(i.b)("inlineCode",{parentName:"p"},"workbench-site.xml"),", but we do not recommend you to modify it.")),Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"The submarine-workbench, IP and 8080 ports that are accessible locally through the port 8080 of 127.0.0.1 are configured via ",Object(i.b)("inlineCode",{parentName:"p"},"workbench-site.xml"),", but we do not recommend you to modify it.")),Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"When you modify the ",Object(i.b)("inlineCode",{parentName:"p"},"angular")," or ",Object(i.b)("inlineCode",{parentName:"p"},"js")," of workbench-web, you need to execute the yarn run build command in the workbench-web directory, and let your modified code update to the dist directory, so that you can see the effect of your code modification in the workbench."))))}s.isMDXComponent=!0},136:function(e,n,t){"use strict";t.d(n,"a",(function(){return u})),t.d(n,"b",(function(){return p}));var r=t(0),a=t.n(r);function i(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function o(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);n&&(r=r.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,r)}return t}function b(e){for(var n=1;n<arguments.length;n++){var t=null!=arguments[n]?arguments[n]:{};n%2?o(Object(t),!0).forEach((function(n){i(e,n,t[n])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(t)):o(Object(t)).forEach((function(n){Object.defineProperty(e,n,Object.getOwnPropertyDescriptor(t,n))}))}return e}function c(e,n){if(null==e)return{};var t,r,a=function(e,n){if(null==e)return{};var t,r,a={},i=Object.keys(e);for(r=0;r<i.length;r++)t=i[r],n.indexOf(t)>=0||(a[t]=e[t]);return a}(e,n);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r<i.length;r++)t=i[r],n.indexOf(t)>=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(a[t]=e[t])}return a}var l=a.a.createContext({}),s=function(e){var n=a.a.useContext(l),t=n;return e&&(t="function"==typeof e?e(n):b(b({},n),e)),t},u=function(e){var n=s(e.components);return a.a.createElement(l.Provider,{value:n},e.children)},d={inlineCode:"code",wrapper:function(e){var n=e.children;return a.a.createElement(a.a.Fragment,{},n)}},m=a.a.forwardRef((function(e,n){var t=e.components,r=e.mdxType,i=e.originalType,o=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),u=s(t),m=r,p=u["".concat(o,".").concat(m)]||u[m]||d[m]||i;return t?a.a.createElement(p,b(b({ref:n},l),{},{components:t})):a.a.createElement(p,b({ref:n},l))}));function p(e,n){var t=arguments,r=n&&n.mdxType;if("string"==typeof e||r){var i=t.length,o=new Array(i);o[0]=m;var b={};for(var c in n)hasOwnProperty.call(n,c)&&(b[c]=n[c]);b.originalType=e,b.mdxType="string"==typeof e?e:r,o[1]=b;for(var l=2;l<i;l++)o[l]=t[l];return a.a.createElement.apply(null,o)}return a.a.createElement.apply(null,t)}m.displayName="MDXCreateElement"},210:function(e,n,t){"use strict";t.r(n),n.default=t.p+"assets/images/Run-Debug-Configuration-f8968b8bcb5346a3602a24afccc8a25f.png"}}]);