| "use strict";(self.webpackChunkwebsite=self.webpackChunkwebsite||[]).push([[2656],{3905:function(e,t,n){n.d(t,{Zo:function(){return u},kt:function(){return k}});var a=n(7294);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function i(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?o(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):o(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,a,r=function(e,t){if(null==e)return{};var n,a,r={},o=Object.keys(e);for(a=0;a<o.length;a++)n=o[a],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a<o.length;a++)n=o[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var p=a.createContext({}),s=function(e){var t=a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},u=function(e){var t=s(e.components);return a.createElement(p.Provider,{value:t},e.children)},m={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},c=a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,o=e.originalType,p=e.parentName,u=l(e,["components","mdxType","originalType","parentName"]),c=s(n),k=r,d=c["".concat(p,".").concat(k)]||c[k]||m[k]||o;return n?a.createElement(d,i(i({ref:t},u),{},{components:n})):a.createElement(d,i({ref:t},u))}));function k(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=n.length,i=new Array(o);i[0]=c;var l={};for(var p in t)hasOwnProperty.call(t,p)&&(l[p]=t[p]);l.originalType=e,l.mdxType="string"==typeof e?e:r,i[1]=l;for(var s=2;s<o;s++)i[s]=n[s];return a.createElement.apply(null,i)}return a.createElement.apply(null,n)}c.displayName="MDXCreateElement"},235:function(e,t,n){n.r(t),n.d(t,{assets:function(){return u},contentTitle:function(){return p},default:function(){return k},frontMatter:function(){return l},metadata:function(){return s},toc:function(){return m}});var a=n(7462),r=n(3366),o=(n(7294),n(3905)),i=["components"],l={title:"Python SDK Development"},p=void 0,s={unversionedId:"userDocs/submarine-sdk/pysubmarine/development",id:"userDocs/submarine-sdk/pysubmarine/development",title:"Python SDK Development",description:"\x3c!---",source:"@site/docs/userDocs/submarine-sdk/pysubmarine/development.md",sourceDirName:"userDocs/submarine-sdk/pysubmarine",slug:"/userDocs/submarine-sdk/pysubmarine/development",permalink:"/docs/next/userDocs/submarine-sdk/pysubmarine/development",editUrl:"https://github.com/apache/submarine/edit/master/website/docs/userDocs/submarine-sdk/pysubmarine/development.md",tags:[],version:"current",frontMatter:{title:"Python SDK Development"}},u={},m=[{value:"Prerequisites",id:"prerequisites",level:3},{value:"PySubmarine Docker",id:"pysubmarine-docker",level:3},{value:"Coding Style",id:"coding-style",level:3},{value:"Unit Testing",id:"unit-testing",level:3},{value:"Generate python SDK from swagger",id:"generate-python-sdk-from-swagger",level:3},{value:"Model Management Model Development",id:"model-management-model-development",level:3},{value:"Upload package to PyPi",id:"upload-package-to-pypi",level:3}],c={toc:m};function k(e){var t=e.components,n=(0,r.Z)(e,i);return(0,o.kt)("wrapper",(0,a.Z)({},c,n,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("p",null,"This page provides general Python development guidelines and source build instructions"),(0,o.kt)("h3",{id:"prerequisites"},"Prerequisites"),(0,o.kt)("p",null,"This is required for developing & testing changes, we recommend installing pysubmarine\nin its own conda environment by running the following"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-bash"},"conda create --name submarine-dev python=3.7\nconda activate submarine-dev\n\n# Install auto-format and lints from current checkout\npip install -r ./dev-support/style-check/python/lint-requirements.txt\n\n# Install mypy from current checkout\npip install -r ./dev-support/style-check/python/mypy-requirements.txt\n\n# test-requirements.txt from current checkout\npip install -r ./submarine-sdk/pysubmarine/github-actions/test-requirements.txt\n\n# Installs pysubmarine from current checkout\npip install -e ./submarine-sdk/pysubmarine\n")),(0,o.kt)("h3",{id:"pysubmarine-docker"},"PySubmarine Docker"),(0,o.kt)("p",null,"We also use docker to provide build environments for CI, development,\ngenerate python sdk from swagger."),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-bash"},"./run-pysubmarine-ci.sh\n")),(0,o.kt)("p",null,"The script does the following things:"),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"Start an interactive bash session"),(0,o.kt)("li",{parentName:"ul"},"Mount submarine directory to /workspace and set it as home"),(0,o.kt)("li",{parentName:"ul"},"Switch user to be the same user that calls the ",(0,o.kt)("inlineCode",{parentName:"li"},"run-pysubmarine-ci.sh"))),(0,o.kt)("h3",{id:"coding-style"},"Coding Style"),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"Use ",(0,o.kt)("a",{parentName:"li",href:"https://github.com/PyCQA/isort"},"isort")," to sort the Python imports and ",(0,o.kt)("a",{parentName:"li",href:"https://github.com/psf/black"},"black")," to format Python code"),(0,o.kt)("li",{parentName:"ul"},"Both style is configured in ",(0,o.kt)("inlineCode",{parentName:"li"},"pyproject.toml")),(0,o.kt)("li",{parentName:"ul"},"To autoformat code")),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-bash"},"./dev-support/style-check/python/auto-format.sh\n")),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"Use ",(0,o.kt)("a",{parentName:"li",href:"https://github.com/PyCQA/flake8"},"flake8")," to verify the linter, its' configure is in ",(0,o.kt)("inlineCode",{parentName:"li"},".flake8"),"."),(0,o.kt)("li",{parentName:"ul"},"Also, we are using ",(0,o.kt)("a",{parentName:"li",href:"https://github.com/python/mypy"},"mypy")," to check the static type in ",(0,o.kt)("inlineCode",{parentName:"li"},"submarine-sdk/pysubmarine/submarine"),"."),(0,o.kt)("li",{parentName:"ul"},"Verify linter pass before submitting a pull request by running:")),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-bash"},"./dev-support/style-check/python/lint.sh\n")),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"If you encouter a unexpected format, use the following method")),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-python"},'# fmt: off\n "Unexpected format, formated by yourself"\n# fmt: on\n')),(0,o.kt)("h3",{id:"unit-testing"},"Unit Testing"),(0,o.kt)("p",null,"We are using ",(0,o.kt)("a",{parentName:"p",href:"https://docs.pytest.org/en/latest/"},"pytest")," to develop our unit test suite.\nAfter building the project (see below) you can run its unit tests like so:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-bash"},"cd submarine-sdk/pysubmarine\n")),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"Run unit test")),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-shell",metastring:"script",script:!0},'pytest --cov=submarine -vs -m "not e2e"\n')),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"Run integration test")),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-shell",metastring:"script",script:!0},'pytest --cov=submarine -vs -m "e2e"\n')),(0,o.kt)("blockquote",null,(0,o.kt)("p",{parentName:"blockquote"},"Before run this command in local, you should make sure the submarine server is running.")),(0,o.kt)("h3",{id:"generate-python-sdk-from-swagger"},"Generate python SDK from swagger"),(0,o.kt)("p",null,"We use ",(0,o.kt)("a",{parentName:"p",href:"https://openapi-generator.tech/docs/installation/#jar"},"open-api generator"),"\nto generate pysubmarine client API that used to communicate with submarine server."),(0,o.kt)("ol",null,(0,o.kt)("li",{parentName:"ol"},(0,o.kt)("p",{parentName:"li"},"To generate different API Component, please change the code in ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/apache/submarine/blob/master/submarine-server/server-core/src/main/java/org/apache/submarine/server/Bootstrap.java"},"Bootstrap.java"),". If just updating java code for ",(0,o.kt)("inlineCode",{parentName:"p"},"NotebookRestApi")," , ",(0,o.kt)("inlineCode",{parentName:"p"},"ExperimentRestApi")," or ",(0,o.kt)("inlineCode",{parentName:"p"},"EnvironmentRestApi"),", please skip step 1."),(0,o.kt)("pre",{parentName:"li"},(0,o.kt)("code",{parentName:"pre",className:"language-java"},'SwaggerConfiguration oasConfig = new SwaggerConfiguration()\n .openAPI(oas)\n .resourcePackages(Stream.of("org.apache.submarine.server.rest")\n .collect(Collectors.toSet()))\n .resourceClasses(Stream.of("org.apache.submarine.server.rest.NotebookRestApi",\n "org.apache.submarine.server.rest.ExperimentRestApi",\n "org.apache.submarine.server.rest.EnvironmentRestApi")\n .collect(Collectors.toSet()));\n')),(0,o.kt)("blockquote",{parentName:"li"},(0,o.kt)("p",{parentName:"blockquote"},"After starting the server, ",(0,o.kt)("inlineCode",{parentName:"p"},"http://localhost:8080/v1/openapi.json")," will includes API specs for ",(0,o.kt)("inlineCode",{parentName:"p"},"NotebookRestApi"),", ",(0,o.kt)("inlineCode",{parentName:"p"},"ExperimentRestApi")," and ",(0,o.kt)("inlineCode",{parentName:"p"},"EnvironmentRestApi"))))),(0,o.kt)("ol",{start:2},(0,o.kt)("li",{parentName:"ol"},(0,o.kt)("p",{parentName:"li"},(0,o.kt)("a",{parentName:"p",href:"https://github.com/apache/submarine/blob/master/dev-support/pysubmarine/swagger_config.json"},"swagger_config.json")," defines the import path for python SDK"),(0,o.kt)("p",{parentName:"li"},"Ex:"),(0,o.kt)("p",{parentName:"li"},"For ",(0,o.kt)("inlineCode",{parentName:"p"},"submarine.client")),(0,o.kt)("pre",{parentName:"li"},(0,o.kt)("code",{parentName:"pre",className:"language-json"},'{\n "packageName" : "submarine.client",\n "projectName" : "submarine.client",\n "packageVersion": "0.8.0-SNAPSHOT"\n}\n')),(0,o.kt)("blockquote",{parentName:"li"},(0,o.kt)("p",{parentName:"blockquote"},"Usage: ",(0,o.kt)("inlineCode",{parentName:"p"},"import submarine.client...")))),(0,o.kt)("li",{parentName:"ol"},(0,o.kt)("p",{parentName:"li"},"Execute ",(0,o.kt)("inlineCode",{parentName:"p"},"./dev-support/pysubmarine/gen-sdk.sh")," to generate latest version of SDK."),(0,o.kt)("blockquote",{parentName:"li"},(0,o.kt)("p",{parentName:"blockquote"},"Notice: Please install required package before running the script: ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/apache/submarine/blob/master/dev-support/style-check/python/lint-requirements.txt"},"lint-requirements.txt")))),(0,o.kt)("li",{parentName:"ol"},(0,o.kt)("p",{parentName:"li"},"In ",(0,o.kt)("inlineCode",{parentName:"p"},"submarine/submarine-sdk/pysubmarine/client/api_client.py")," line 74"),(0,o.kt)("p",{parentName:"li"},"Please change"),(0,o.kt)("pre",{parentName:"li"},(0,o.kt)("code",{parentName:"pre",className:"language-python"},'"long": int if six.PY3 else long, # noqa: F821\n')),(0,o.kt)("p",{parentName:"li"},"to"),(0,o.kt)("pre",{parentName:"li"},(0,o.kt)("code",{parentName:"pre",className:"language-python"},'"long": int,\n')))),(0,o.kt)("h3",{id:"model-management-model-development"},"Model Management Model Development"),(0,o.kt)("p",null,"For local development, we can access cluster's service easily thanks to ",(0,o.kt)("a",{parentName:"p",href:"https://www.telepresence.io/"},"telepresence"),".\nTo elaborate, we can develop the sdk in local but can reach out to database and minio server by proxy."),(0,o.kt)("ol",null,(0,o.kt)("li",{parentName:"ol"},"Install telepresence follow ",(0,o.kt)("a",{parentName:"li",href:"https://www.telepresence.io/reference/install"},"the instruction"),"."),(0,o.kt)("li",{parentName:"ol"},"Start proxy pod")),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"telepresence --new-deployment submarine-dev\n")),(0,o.kt)("ol",{start:3},(0,o.kt)("li",{parentName:"ol"},"You can develop as if in the cluster.")),(0,o.kt)("h3",{id:"upload-package-to-pypi"},"Upload package to PyPi"),(0,o.kt)("p",null,"For Apache Submarine committer and PMCs to do a new release."),(0,o.kt)("ol",null,(0,o.kt)("li",{parentName:"ol"},"Change the version from 0.x.x-SNAPSHOT to 0.x.x\nin ",(0,o.kt)("a",{parentName:"li",href:"https://github.com/apache/submarine/blob/master/submarine-sdk/pysubmarine/setup.py"},"setup.py")),(0,o.kt)("li",{parentName:"ol"},"Install Python packages")),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-bash"},"cd submarine-sdk/pysubmarine\npip install -r github-actions/pypi-requirements.txt\n")),(0,o.kt)("ol",{start:3},(0,o.kt)("li",{parentName:"ol"},"Compiling Your Package")),(0,o.kt)("p",null,"It will create ",(0,o.kt)("inlineCode",{parentName:"p"},"build"),", ",(0,o.kt)("inlineCode",{parentName:"p"},"dist"),", and ",(0,o.kt)("inlineCode",{parentName:"p"},"project.egg.info"),"\nin your local directory"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-bash"},"python setup.py bdist_wheel\n")),(0,o.kt)("ol",{start:4},(0,o.kt)("li",{parentName:"ol"},"Upload python package to TestPyPI for testing")),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-bash"},"python -m twine upload --repository testpypi dist/*\n")),(0,o.kt)("ol",{start:5},(0,o.kt)("li",{parentName:"ol"},"Upload python package to PyPi")),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-bash"},"python -m twine upload --repository-url https://upload.pypi.org/legacy/ dist/*\n")))}k.isMDXComponent=!0}}]); |