(window.webpackJsonp=window.webpackJsonp||[]).push([[65],{132:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return o})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return b}));var r=n(3),a=n(7),i=(n(0),n(144)),o={title:"Python SDK Development"},l={unversionedId:"userDocs/submarine-sdk/pysubmarine/development",id:"userDocs/submarine-sdk/pysubmarine/development",isDocsHomePage:!1,title:"Python SDK Development",description:"\x3c!---",source:"@site/docs/userDocs/submarine-sdk/pysubmarine/development.md",slug:"/userDocs/submarine-sdk/pysubmarine/development",permalink:"/docs/userDocs/submarine-sdk/pysubmarine/development",editUrl:"https://github.com/apache/submarine/edit/master/website/docs/userDocs/submarine-sdk/pysubmarine/development.md",version:"current"},s=[{value:"Prerequisites",id:"prerequisites",children:[]},{value:"PySubmarine Docker",id:"pysubmarine-docker",children:[]},{value:"Coding Style",id:"coding-style",children:[]},{value:"Unit Testing",id:"unit-testing",children:[]},{value:"Generate python SDK from swagger",id:"generate-python-sdk-from-swagger",children:[]},{value:"Model Management Model Development",id:"model-management-model-development",children:[]},{value:"Upload package to PyPi",id:"upload-package-to-pypi",children:[]}],c={toc:s};function b(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("p",null,"This page provides general Python development guidelines and source build instructions"),Object(i.b)("h3",{id:"prerequisites"},"Prerequisites"),Object(i.b)("p",null,"This is required for developing & testing changes, we recommend installing pysubmarine\nin its own conda environment by running the following"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-bash"},"conda create --name submarine-dev python=3.6\nconda activate submarine-dev\n\n# lint-requirements.txt and test-requirements.txt are in ./submarine-sdk/pysubmarine/github-actions\npip install -r lint-requirements.txt\npip install -r test-requirements.txt\n\n# Installs pysubmarine from current checkout\npip install ./submarine-sdk/pysubmarine\n")),Object(i.b)("h3",{id:"pysubmarine-docker"},"PySubmarine Docker"),Object(i.b)("p",null,"We also use docker to provide build environments for CI, development,\ngenerate python sdk from swagger."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-bash"},"./run-pysubmarine-ci.sh\n")),Object(i.b)("p",null,"The script does the following things:"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"Start an interactive bash session"),Object(i.b)("li",{parentName:"ul"},"Mount submarine directory to /workspace and set it as home"),Object(i.b)("li",{parentName:"ul"},"Switch user to be the same user that calls the ",Object(i.b)("inlineCode",{parentName:"li"},"run-pysubmarine-ci.sh"))),Object(i.b)("h3",{id:"coding-style"},"Coding Style"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"Use ",Object(i.b)("a",{parentName:"li",href:"https://github.com/google/yapf"},"yapf")," to format Python code"),Object(i.b)("li",{parentName:"ul"},"yapf style is configured in ",Object(i.b)("inlineCode",{parentName:"li"},".style.yapf")," file"),Object(i.b)("li",{parentName:"ul"},"To autoformat code")),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-bash"},"./submarine-sdk/pysubmarine/github-actions/auto-format.sh\n")),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"Verify linter pass before submitting a pull request by running:")),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-bash"},"./submarine-sdk/pysubmarine/github-actions/lint.sh\n")),Object(i.b)("h3",{id:"unit-testing"},"Unit Testing"),Object(i.b)("p",null,"We are using ",Object(i.b)("a",{parentName:"p",href:"https://docs.pytest.org/en/latest/"},"pytest")," to develop our unit test suite.\nAfter building the project (see below) you can run its unit tests like so:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-bash"},"cd submarine-sdk/pysubmarine\n")),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"Run unit test")),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-shell",metastring:"script",script:!0},'pytest --cov=submarine -vs -m "not e2e"\n')),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"Run integration test")),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-shell",metastring:"script",script:!0},'pytest --cov=submarine -vs -m "e2e"\n')),Object(i.b)("blockquote",null,Object(i.b)("p",{parentName:"blockquote"},"Before run this command in local, you should make sure the submarine server is running.")),Object(i.b)("h3",{id:"generate-python-sdk-from-swagger"},"Generate python SDK from swagger"),Object(i.b)("p",null,"We use ",Object(i.b)("a",{parentName:"p",href:"https://openapi-generator.tech/docs/installation/#jar"},"open-api generator"),"\nto generate pysubmarine client API that used to communicate with submarine server."),Object(i.b)("p",null,"If change below files, please run ",Object(i.b)("inlineCode",{parentName:"p"},"./dev-support/pysubmarine/gen-sdk.sh"),"\nto generate latest version of SDK."),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},Object(i.b)("a",{parentName:"li",href:"https://github.com/apache/submarine/blob/master/submarine-server/server-core/src/main/java/org/apache/submarine/server/Bootstrap.java"},"Bootstrap.java")),Object(i.b)("li",{parentName:"ul"},Object(i.b)("a",{parentName:"li",href:"https://github.com/apache/submarine/blob/master/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/ExperimentRestApi.java"},"ExperimentRestApi.java"))),Object(i.b)("h3",{id:"model-management-model-development"},"Model Management Model Development"),Object(i.b)("p",null,"For local development, we can access cluster's service easily thanks to ",Object(i.b)("a",{parentName:"p",href:"https://www.telepresence.io/"},"telepresence"),".\nTo elaborate, we can develop the sdk in local but can reach out to mlflow server by proxy."),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"Install telepresence follow ",Object(i.b)("a",{parentName:"li",href:"https://www.telepresence.io/reference/install"},"the instruction"),"."),Object(i.b)("li",{parentName:"ol"},"Start proxy pod")),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"telepresence --new-deployment submarine-dev\n")),Object(i.b)("ol",{start:3},Object(i.b)("li",{parentName:"ol"},"You can develop as if in the cluster.")),Object(i.b)("h3",{id:"upload-package-to-pypi"},"Upload package to PyPi"),Object(i.b)("p",null,"For Apache Submarine committer and PMCs to do a new release."),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"Change the version from 0.x.x-SNAPSHOT to 0.x.x\nin ",Object(i.b)("a",{parentName:"li",href:"https://github.com/apache/submarine/blob/master/submarine-sdk/pysubmarine/setup.py"},"setup.py")),Object(i.b)("li",{parentName:"ol"},"Install Python packages")),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-bash"},"cd submarine-sdk/pysubmarine\npip install -r github-actions/pypi-requirements.txt\n")),Object(i.b)("ol",{start:3},Object(i.b)("li",{parentName:"ol"},"Compiling Your Package")),Object(i.b)("p",null,"It will create ",Object(i.b)("inlineCode",{parentName:"p"},"build"),", ",Object(i.b)("inlineCode",{parentName:"p"},"dist"),", and ",Object(i.b)("inlineCode",{parentName:"p"},"project.egg.info"),"\nin your local directory"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-bash"},"python setup.py bdist_wheel\n")),Object(i.b)("ol",{start:4},Object(i.b)("li",{parentName:"ol"},"Upload python package to TestPyPI for testing")),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-bash"},"python -m twine upload --repository testpypi dist/*\n")),Object(i.b)("ol",{start:5},Object(i.b)("li",{parentName:"ol"},"Upload python package to PyPi")),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-bash"},"python -m twine upload --repository-url https://upload.pypi.org/legacy/ dist/*\n")))}b.isMDXComponent=!0},144:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return d}));var r=n(0),a=n.n(r);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?o(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):o(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,a=function(e,t){if(null==e)return{};var n,r,a={},i=Object.keys(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),b=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},p=function(e){var t=b(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},m=a.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,o=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=b(n),m=r,d=p["".concat(o,".").concat(m)]||p[m]||u[m]||i;return n?a.a.createElement(d,l(l({ref:t},c),{},{components:n})):a.a.createElement(d,l({ref:t},c))}));function d(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,o=new Array(i);o[0]=m;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:r,o[1]=l;for(var c=2;c<i;c++)o[c]=n[c];return a.a.createElement.apply(null,o)}return a.a.createElement.apply(null,n)}m.displayName="MDXCreateElement"}}]);