| (window.webpackJsonp=window.webpackJsonp||[]).push([[70],{137:function(e,n,r){"use strict";r.r(n),r.d(n,"frontMatter",(function(){return o})),r.d(n,"metadata",(function(){return c})),r.d(n,"toc",(function(){return s})),r.d(n,"default",(function(){return b}));var t=r(3),i=r(7),a=(r(0),r(144)),o={title:"Project Architecture"},c={unversionedId:"devDocs/README",id:"devDocs/README",isDocsHomePage:!1,title:"Project Architecture",description:"\x3c!--",source:"@site/docs/devDocs/README.md",slug:"/devDocs/README",permalink:"/docs/devDocs/README",editUrl:"https://github.com/apache/submarine/edit/master/website/docs/devDocs/README.md",version:"current",sidebar:"docs",previous:{title:"Running Submarine on YARN",permalink:"/docs/adminDocs/yarn/README"},next:{title:"Dependencies for Submarine",permalink:"/docs/devDocs/Dependencies"}},s=[{value:"1. Introduction",id:"1-introduction",children:[]},{value:"2. Submarine Project Structure",id:"2-submarine-project-structure",children:[{value:"2.1. submarine-client",id:"21-submarine-client",children:[]},{value:"2.2. submarine-cloud-v2",id:"22-submarine-cloud-v2",children:[]},{value:"2.3. submarine-commons",id:"23-submarine-commons",children:[]},{value:"2.4. submarine-dist",id:"24-submarine-dist",children:[]},{value:"2.5. submarine-sdk",id:"25-submarine-sdk",children:[]},{value:"2.6. submarine-security",id:"26-submarine-security",children:[]},{value:"2.7. submarine-server",id:"27-submarine-server",children:[]},{value:"2.8. submarine-test",id:"28-submarine-test",children:[]},{value:"2.9. submarine-workbench",id:"29-submarine-workbench",children:[]},{value:"2.10 dev-support",id:"210-dev-support",children:[]}]}],u={toc:s};function b(e){var n=e.components,r=Object(i.a)(e,["components"]);return Object(a.b)("wrapper",Object(t.a)({},u,r,{components:n,mdxType:"MDXLayout"}),Object(a.b)("h2",{id:"1-introduction"},"1. Introduction"),Object(a.b)("p",null,"This document mainly describes the structure of each module of the Submarine project, the development and test description of each module."),Object(a.b)("h2",{id:"2-submarine-project-structure"},"2. Submarine Project Structure"),Object(a.b)("h3",{id:"21-submarine-client"},"2.1. submarine-client"),Object(a.b)("p",null,"Provide the CLI interface for submarine user. (Currently only support YARN service)"),Object(a.b)("h3",{id:"22-submarine-cloud-v2"},"2.2. submarine-cloud-v2"),Object(a.b)("p",null,"The operator for Submarine application. For details, please see the ",Object(a.b)("a",{parentName:"p",href:"https://github.com/apache/submarine/blob/master/submarine-cloud-v2/README.md"},"README on github"),"."),Object(a.b)("h3",{id:"23-submarine-commons"},"2.3. submarine-commons"),Object(a.b)("p",null,"Define utility function used in multiple packages, mainly related to hadoop."),Object(a.b)("h3",{id:"24-submarine-dist"},"2.4. submarine-dist"),Object(a.b)("p",null,"Store the pre-release files."),Object(a.b)("h3",{id:"25-submarine-sdk"},"2.5. submarine-sdk"),Object(a.b)("p",null,"Provide Python SDK for submarine user."),Object(a.b)("h3",{id:"26-submarine-security"},"2.6. submarine-security"),Object(a.b)("p",null,"Provide authorization for Apache Spark to talking to Ranger Admin."),Object(a.b)("h3",{id:"27-submarine-server"},"2.7. submarine-server"),Object(a.b)("p",null,"Include core server, restful api, and k8s/yarn submitter."),Object(a.b)("h3",{id:"28-submarine-test"},"2.8. submarine-test"),Object(a.b)("p",null,"Provide end-to-end and k8s test for submarine."),Object(a.b)("h3",{id:"29-submarine-workbench"},"2.9. submarine-workbench"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"workbench-server: is a Jetty-based web server service. Workbench-server provides RESTful interface and Websocket interface. The RESTful interface provides workbench-web with management capabilities for databases such as project, department, user, and role."),Object(a.b)("li",{parentName:"ul"},"workbench-web: is a web front-end service based on Angular.js framework. With workbench-web users can manage Submarine project, department, user, role through browser. You can also use the notebook to develop machine learning algorithms, model release and other lifecycle management.")),Object(a.b)("h3",{id:"210-dev-support"},"2.10 dev-support"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},Object(a.b)("strong",{parentName:"li"},"mini-submarine"),": by using the docker image provided by Submarine, you can\nexperience all the functions of Submarine in a single docker environment, while\nmini-submarine also provides developers with a development and testing\nenvironment, Avoid the hassle of installing and deploying the runtime\nenvironment."),Object(a.b)("li",{parentName:"ul"},Object(a.b)("strong",{parentName:"li"},"submarine-installer"),": submarine-installer is our submarine runtime\nenvironment installation tool for yarn-3.1+ and above.By using\nsubmarine-installer, it is easy to install and deploy system services such as\n",Object(a.b)("inlineCode",{parentName:"li"},"docker"),", ",Object(a.b)("inlineCode",{parentName:"li"},"nvidia-docker"),", ",Object(a.b)("inlineCode",{parentName:"li"},"nvidia driver"),", ",Object(a.b)("inlineCode",{parentName:"li"},"ETCD"),", ",Object(a.b)("inlineCode",{parentName:"li"},"Calico network")," etc.\nrequired by yarn-3.1+.")))}b.isMDXComponent=!0},144:function(e,n,r){"use strict";r.d(n,"a",(function(){return l})),r.d(n,"b",(function(){return p}));var t=r(0),i=r.n(t);function a(e,n,r){return n in e?Object.defineProperty(e,n,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[n]=r,e}function o(e,n){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var t=Object.getOwnPropertySymbols(e);n&&(t=t.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),r.push.apply(r,t)}return r}function c(e){for(var n=1;n<arguments.length;n++){var r=null!=arguments[n]?arguments[n]:{};n%2?o(Object(r),!0).forEach((function(n){a(e,n,r[n])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(r)):o(Object(r)).forEach((function(n){Object.defineProperty(e,n,Object.getOwnPropertyDescriptor(r,n))}))}return e}function s(e,n){if(null==e)return{};var r,t,i=function(e,n){if(null==e)return{};var r,t,i={},a=Object.keys(e);for(t=0;t<a.length;t++)r=a[t],n.indexOf(r)>=0||(i[r]=e[r]);return i}(e,n);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(t=0;t<a.length;t++)r=a[t],n.indexOf(r)>=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(i[r]=e[r])}return i}var u=i.a.createContext({}),b=function(e){var n=i.a.useContext(u),r=n;return e&&(r="function"==typeof e?e(n):c(c({},n),e)),r},l=function(e){var n=b(e.components);return i.a.createElement(u.Provider,{value:n},e.children)},d={inlineCode:"code",wrapper:function(e){var n=e.children;return i.a.createElement(i.a.Fragment,{},n)}},m=i.a.forwardRef((function(e,n){var r=e.components,t=e.mdxType,a=e.originalType,o=e.parentName,u=s(e,["components","mdxType","originalType","parentName"]),l=b(r),m=t,p=l["".concat(o,".").concat(m)]||l[m]||d[m]||a;return r?i.a.createElement(p,c(c({ref:n},u),{},{components:r})):i.a.createElement(p,c({ref:n},u))}));function p(e,n){var r=arguments,t=n&&n.mdxType;if("string"==typeof e||t){var a=r.length,o=new Array(a);o[0]=m;var c={};for(var s in n)hasOwnProperty.call(n,s)&&(c[s]=n[s]);c.originalType=e,c.mdxType="string"==typeof e?e:t,o[1]=c;for(var u=2;u<a;u++)o[u]=r[u];return i.a.createElement.apply(null,o)}return i.a.createElement.apply(null,r)}m.displayName="MDXCreateElement"}}]); |