| (window.webpackJsonp=window.webpackJsonp||[]).push([[102],{169:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return p})),n.d(t,"default",(function(){return c}));var a=n(3),r=n(7),o=(n(0),n(193)),i={title:"Python SDK Development"},l={unversionedId:"userDocs/submarine-sdk/pysubmarine/development",id:"userDocs/submarine-sdk/pysubmarine/development",isDocsHomePage:!1,title:"Python SDK Development",description:"\x3c!---",source:"@site/docs/userDocs/submarine-sdk/pysubmarine/development.md",slug:"/userDocs/submarine-sdk/pysubmarine/development",permalink:"/docs/next/userDocs/submarine-sdk/pysubmarine/development",editUrl:"https://github.com/apache/submarine/edit/master/website/docs/userDocs/submarine-sdk/pysubmarine/development.md",version:"current"},p=[{value:"Prerequisites",id:"prerequisites",children:[]},{value:"PySubmarine Docker",id:"pysubmarine-docker",children:[]},{value:"Coding Style",id:"coding-style",children:[]},{value:"Unit Testing",id:"unit-testing",children:[]},{value:"Generate python SDK from swagger",id:"generate-python-sdk-from-swagger",children:[]},{value:"Model Management Model Development",id:"model-management-model-development",children:[]},{value:"Upload package to PyPi",id:"upload-package-to-pypi",children:[]}],s={toc:p};function c(e){var t=e.components,n=Object(r.a)(e,["components"]);return Object(o.b)("wrapper",Object(a.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("p",null,"This page provides general Python development guidelines and source build instructions"),Object(o.b)("h3",{id:"prerequisites"},"Prerequisites"),Object(o.b)("p",null,"This is required for developing & testing changes, we recommend installing pysubmarine\nin its own conda environment by running the following"),Object(o.b)("pre",null,Object(o.b)("code",{parentName:"pre",className:"language-bash"},"conda create --name submarine-dev python=3.6\nconda activate submarine-dev\n\n# Install auto-format and lints from current checkout\npip install -r ./dev-support/style-check/python/lint-requirements.txt\n\n# Install mypy from current checkout\npip install -r ./dev-support/style-check/python/mypy-requirements.txt\n\n# test-requirements.txt from current checkout\npip install -r ./submarine-sdk/pysubmarine/github-actions/test-requirements.txt\n\n# Installs pysubmarine from current checkout\npip install -e ./submarine-sdk/pysubmarine\n")),Object(o.b)("h3",{id:"pysubmarine-docker"},"PySubmarine Docker"),Object(o.b)("p",null,"We also use docker to provide build environments for CI, development,\ngenerate python sdk from swagger."),Object(o.b)("pre",null,Object(o.b)("code",{parentName:"pre",className:"language-bash"},"./run-pysubmarine-ci.sh\n")),Object(o.b)("p",null,"The script does the following things:"),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"Start an interactive bash session"),Object(o.b)("li",{parentName:"ul"},"Mount submarine directory to /workspace and set it as home"),Object(o.b)("li",{parentName:"ul"},"Switch user to be the same user that calls the ",Object(o.b)("inlineCode",{parentName:"li"},"run-pysubmarine-ci.sh"))),Object(o.b)("h3",{id:"coding-style"},"Coding Style"),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"Use ",Object(o.b)("a",{parentName:"li",href:"https://github.com/PyCQA/isort"},"isort")," to sort the Python imports and ",Object(o.b)("a",{parentName:"li",href:"https://github.com/psf/black"},"black")," to format Python code"),Object(o.b)("li",{parentName:"ul"},"Both style is configured in ",Object(o.b)("inlineCode",{parentName:"li"},"pyproject.toml")),Object(o.b)("li",{parentName:"ul"},"To autoformat code")),Object(o.b)("pre",null,Object(o.b)("code",{parentName:"pre",className:"language-bash"},"./dev-support/style-check/python/auto-format.sh\n")),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"Use ",Object(o.b)("a",{parentName:"li",href:"https://github.com/PyCQA/flake8"},"flake8")," to verify the linter, its' configure is in ",Object(o.b)("inlineCode",{parentName:"li"},".flake8"),"."),Object(o.b)("li",{parentName:"ul"},"Also, we are using ",Object(o.b)("a",{parentName:"li",href:"https://github.com/python/mypy"},"mypy")," to check the static type in ",Object(o.b)("inlineCode",{parentName:"li"},"submarine-sdk/pysubmarine/submarine"),"."),Object(o.b)("li",{parentName:"ul"},"Verify linter pass before submitting a pull request by running:")),Object(o.b)("pre",null,Object(o.b)("code",{parentName:"pre",className:"language-bash"},"./dev-support/style-check/python/lint.sh\n")),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"If you encouter a unexpected format, use the following method")),Object(o.b)("pre",null,Object(o.b)("code",{parentName:"pre",className:"language-python"},'# fmt: off\n "Unexpected format, formated by yourself"\n# fmt: on\n')),Object(o.b)("h3",{id:"unit-testing"},"Unit Testing"),Object(o.b)("p",null,"We are using ",Object(o.b)("a",{parentName:"p",href:"https://docs.pytest.org/en/latest/"},"pytest")," to develop our unit test suite.\nAfter building the project (see below) you can run its unit tests like so:"),Object(o.b)("pre",null,Object(o.b)("code",{parentName:"pre",className:"language-bash"},"cd submarine-sdk/pysubmarine\n")),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"Run unit test")),Object(o.b)("pre",null,Object(o.b)("code",{parentName:"pre",className:"language-shell",metastring:"script",script:!0},'pytest --cov=submarine -vs -m "not e2e"\n')),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"Run integration test")),Object(o.b)("pre",null,Object(o.b)("code",{parentName:"pre",className:"language-shell",metastring:"script",script:!0},'pytest --cov=submarine -vs -m "e2e"\n')),Object(o.b)("blockquote",null,Object(o.b)("p",{parentName:"blockquote"},"Before run this command in local, you should make sure the submarine server is running.")),Object(o.b)("h3",{id:"generate-python-sdk-from-swagger"},"Generate python SDK from swagger"),Object(o.b)("p",null,"We use ",Object(o.b)("a",{parentName:"p",href:"https://openapi-generator.tech/docs/installation/#jar"},"open-api generator"),"\nto generate pysubmarine client API that used to communicate with submarine server."),Object(o.b)("ol",null,Object(o.b)("li",{parentName:"ol"},Object(o.b)("p",{parentName:"li"},"To generate different API Component, please change the code in ",Object(o.b)("a",{parentName:"p",href:"https://github.com/apache/submarine/blob/master/submarine-server/server-core/src/main/java/org/apache/submarine/server/Bootstrap.java"},"Bootstrap.java"),". If just updating java code for ",Object(o.b)("inlineCode",{parentName:"p"},"NotebookRestApi")," , ",Object(o.b)("inlineCode",{parentName:"p"},"ExperimentRestApi")," or ",Object(o.b)("inlineCode",{parentName:"p"},"EnvironmentRestApi"),", please skip step 1."),Object(o.b)("pre",{parentName:"li"},Object(o.b)("code",{parentName:"pre",className:"language-java"},'SwaggerConfiguration oasConfig = new SwaggerConfiguration()\n .openAPI(oas)\n .resourcePackages(Stream.of("org.apache.submarine.server.rest")\n .collect(Collectors.toSet()))\n .resourceClasses(Stream.of("org.apache.submarine.server.rest.NotebookRestApi",\n "org.apache.submarine.server.rest.ExperimentRestApi",\n "org.apache.submarine.server.rest.EnvironmentRestApi")\n .collect(Collectors.toSet()));\n')),Object(o.b)("blockquote",{parentName:"li"},Object(o.b)("p",{parentName:"blockquote"},"After starting the server, ",Object(o.b)("inlineCode",{parentName:"p"},"http://localhost:8080/v1/openapi.json")," will includes API specs for ",Object(o.b)("inlineCode",{parentName:"p"},"NotebookRestApi"),", ",Object(o.b)("inlineCode",{parentName:"p"},"ExperimentRestApi")," and ",Object(o.b)("inlineCode",{parentName:"p"},"EnvironmentRestApi"))))),Object(o.b)("ol",{start:2},Object(o.b)("li",{parentName:"ol"},Object(o.b)("p",{parentName:"li"},Object(o.b)("a",{parentName:"p",href:"https://github.com/apache/submarine/blob/master/dev-support/pysubmarine/swagger_config.json"},"swagger_config.json")," defines the import path for python SDK"),Object(o.b)("p",{parentName:"li"},"Ex:"),Object(o.b)("p",{parentName:"li"},"For ",Object(o.b)("inlineCode",{parentName:"p"},"submarine.client")),Object(o.b)("pre",{parentName:"li"},Object(o.b)("code",{parentName:"pre",className:"language-json"},'{\n "packageName" : "submarine.client",\n "projectName" : "submarine.client",\n "packageVersion": "0.7.0"\n}\n')),Object(o.b)("blockquote",{parentName:"li"},Object(o.b)("p",{parentName:"blockquote"},"Usage: ",Object(o.b)("inlineCode",{parentName:"p"},"import submarine.client...")))),Object(o.b)("li",{parentName:"ol"},Object(o.b)("p",{parentName:"li"},"Execute ",Object(o.b)("inlineCode",{parentName:"p"},"./dev-support/pysubmarine/gen-sdk.sh")," to generate latest version of SDK."),Object(o.b)("blockquote",{parentName:"li"},Object(o.b)("p",{parentName:"blockquote"},"Notice: Please install required package before running the script: ",Object(o.b)("a",{parentName:"p",href:"https://github.com/apache/submarine/blob/master/dev-support/style-check/python/lint-requirements.txt"},"lint-requirements.txt")))),Object(o.b)("li",{parentName:"ol"},Object(o.b)("p",{parentName:"li"},"In ",Object(o.b)("inlineCode",{parentName:"p"},"submarine/submarine-sdk/pysubmarine/client/api_client.py")," line 74"),Object(o.b)("p",{parentName:"li"},"Please change"),Object(o.b)("pre",{parentName:"li"},Object(o.b)("code",{parentName:"pre",className:"language-python"},'"long": int if six.PY3 else long, # noqa: F821\n')),Object(o.b)("p",{parentName:"li"},"to"),Object(o.b)("pre",{parentName:"li"},Object(o.b)("code",{parentName:"pre",className:"language-python"},'"long": int,\n')))),Object(o.b)("h3",{id:"model-management-model-development"},"Model Management Model Development"),Object(o.b)("p",null,"For local development, we can access cluster's service easily thanks to ",Object(o.b)("a",{parentName:"p",href:"https://www.telepresence.io/"},"telepresence"),".\nTo elaborate, we can develop the sdk in local but can reach out to database and minio server by proxy."),Object(o.b)("ol",null,Object(o.b)("li",{parentName:"ol"},"Install telepresence follow ",Object(o.b)("a",{parentName:"li",href:"https://www.telepresence.io/reference/install"},"the instruction"),"."),Object(o.b)("li",{parentName:"ol"},"Start proxy pod")),Object(o.b)("pre",null,Object(o.b)("code",{parentName:"pre"},"telepresence --new-deployment submarine-dev\n")),Object(o.b)("ol",{start:3},Object(o.b)("li",{parentName:"ol"},"You can develop as if in the cluster.")),Object(o.b)("h3",{id:"upload-package-to-pypi"},"Upload package to PyPi"),Object(o.b)("p",null,"For Apache Submarine committer and PMCs to do a new release."),Object(o.b)("ol",null,Object(o.b)("li",{parentName:"ol"},"Change the version from 0.x.x-SNAPSHOT to 0.x.x\nin ",Object(o.b)("a",{parentName:"li",href:"https://github.com/apache/submarine/blob/master/submarine-sdk/pysubmarine/setup.py"},"setup.py")),Object(o.b)("li",{parentName:"ol"},"Install Python packages")),Object(o.b)("pre",null,Object(o.b)("code",{parentName:"pre",className:"language-bash"},"cd submarine-sdk/pysubmarine\npip install -r github-actions/pypi-requirements.txt\n")),Object(o.b)("ol",{start:3},Object(o.b)("li",{parentName:"ol"},"Compiling Your Package")),Object(o.b)("p",null,"It will create ",Object(o.b)("inlineCode",{parentName:"p"},"build"),", ",Object(o.b)("inlineCode",{parentName:"p"},"dist"),", and ",Object(o.b)("inlineCode",{parentName:"p"},"project.egg.info"),"\nin your local directory"),Object(o.b)("pre",null,Object(o.b)("code",{parentName:"pre",className:"language-bash"},"python setup.py bdist_wheel\n")),Object(o.b)("ol",{start:4},Object(o.b)("li",{parentName:"ol"},"Upload python package to TestPyPI for testing")),Object(o.b)("pre",null,Object(o.b)("code",{parentName:"pre",className:"language-bash"},"python -m twine upload --repository testpypi dist/*\n")),Object(o.b)("ol",{start:5},Object(o.b)("li",{parentName:"ol"},"Upload python package to PyPi")),Object(o.b)("pre",null,Object(o.b)("code",{parentName:"pre",className:"language-bash"},"python -m twine upload --repository-url https://upload.pypi.org/legacy/ dist/*\n")))}c.isMDXComponent=!0},193:function(e,t,n){"use strict";n.d(t,"a",(function(){return b})),n.d(t,"b",(function(){return d}));var a=n(0),r=n.n(a);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function p(e,t){if(null==e)return{};var n,a,r=function(e,t){if(null==e)return{};var n,a,r={},o=Object.keys(e);for(a=0;a<o.length;a++)n=o[a],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a<o.length;a++)n=o[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var s=r.a.createContext({}),c=function(e){var t=r.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},b=function(e){var t=c(e.components);return r.a.createElement(s.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},m=r.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,o=e.originalType,i=e.parentName,s=p(e,["components","mdxType","originalType","parentName"]),b=c(n),m=a,d=b["".concat(i,".").concat(m)]||b[m]||u[m]||o;return n?r.a.createElement(d,l(l({ref:t},s),{},{components:n})):r.a.createElement(d,l({ref:t},s))}));function d(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=n.length,i=new Array(o);i[0]=m;var l={};for(var p in t)hasOwnProperty.call(t,p)&&(l[p]=t[p]);l.originalType=e,l.mdxType="string"==typeof e?e:a,i[1]=l;for(var s=2;s<o;s++)i[s]=n[s];return r.a.createElement.apply(null,i)}return r.a.createElement.apply(null,n)}m.displayName="MDXCreateElement"}}]); |