| <!doctype html> |
| <html lang="en"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="viewport" content="width=device-width,initial-scale=1"> |
| <meta name="generator" content="Docusaurus v2.0.0-alpha.70"> |
| <link rel="alternate" type="application/rss+xml" href="/blog/rss.xml" title="Apache Submarine Blog RSS Feed"> |
| <link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Submarine Blog Atom Feed"><title data-react-helmet="true">Environments Implementation | Apache Submarine</title><meta data-react-helmet="true" name="twitter:card" content="summary_large_image"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="current"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-current"><meta data-react-helmet="true" property="og:title" content="Environments Implementation | Apache Submarine"><meta data-react-helmet="true" name="description" content="<!--"><meta data-react-helmet="true" property="og:description" content="<!--"><meta data-react-helmet="true" property="og:url" content="https://submarine.apache.org//docs/next/designDocs/environments-implementation"><link data-react-helmet="true" rel="shortcut icon" href="/img/submarine.ico"><link data-react-helmet="true" rel="canonical" href="https://submarine.apache.org//docs/next/designDocs/environments-implementation"><link rel="stylesheet" href="/styles.39775f96.css"> |
| <link rel="preload" href="/styles.6ddab7ad.js" as="script"> |
| <link rel="preload" href="/runtime~main.943b9f3a.js" as="script"> |
| <link rel="preload" href="/main.92db4fb0.js" as="script"> |
| <link rel="preload" href="/1.dab80c77.js" as="script"> |
| <link rel="preload" href="/2.6a1a8326.js" as="script"> |
| <link rel="preload" href="/1f391b9e.f22e8160.js" as="script"> |
| <link rel="preload" href="/128.10c43f42.js" as="script"> |
| <link rel="preload" href="/935f2afb.9989d552.js" as="script"> |
| <link rel="preload" href="/17896441.be8f95bf.js" as="script"> |
| <link rel="preload" href="/a7e506f8.92088415.js" as="script"> |
| </head> |
| <body> |
| <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus"> |
| <nav aria-label="Skip navigation links"><button type="button" tabindex="0" class="skipToContent_11B0">Skip to main content</button></nav><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><div aria-label="Navigation bar toggle" class="navbar__toggle" role="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></div><a class="navbar__brand" href="/"><img src="/img/icons/128.png" alt="Apache Submarine Site Logo" class="themedImage_YANc themedImage--light_3CMI navbar__logo"><img src="/img/icons/128.png" alt="Apache Submarine Site Logo" class="themedImage_YANc themedImage--dark_3ARp navbar__logo"><strong class="navbar__title">Apache Submarine</strong></a><a class="navbar__item navbar__link navbar__link--active" href="/docs/next/gettingStarted/quickstart">Docs</a><a class="navbar__item navbar__link" href="/docs/next/api/environment">API</a><a class="navbar__item navbar__link" href="/docs/next/download">Download</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/docs/next/">master 🏃</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/docs/next/designDocs/environments-implementation">master 🏃</a></li><li><a class="dropdown__link" href="/docs/designDocs/environments-implementation">0.6.0</a></li><li><a class="dropdown__link" href="/versions">All versions</a></li></ul></div><a href="https://github.com/apache/submarine" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link">Apache</a><ul class="dropdown__menu"><li><a href="https://www.apache.org/foundation/how-it-works.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Apache Software Foundation</a></li><li><a href="https://www.apache.org/events/current-event" target="_blank" rel="noopener noreferrer" class="dropdown__link">Events</a></li><li><a href="https://www.apache.org/licenses/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Apache License</a></li><li><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Thanks</a></li><li><a href="https://www.apache.org/security/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Security</a></li><li><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Sponsorship</a></li></ul></div><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_2N3Q"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_3NWk">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_3NWk">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div><div class="navbar__search"><span aria-label="expand searchbar" role="button" class="search-icon" tabindex="0"></span><input type="search" id="search_input_react" placeholder="Search" aria-label="Search" class="navbar__search-input search-bar"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/"><img src="/img/icons/128.png" alt="Apache Submarine Site Logo" class="themedImage_YANc themedImage--light_3CMI navbar__logo"><img src="/img/icons/128.png" alt="Apache Submarine Site Logo" class="themedImage_YANc themedImage--dark_3ARp navbar__logo"><strong class="navbar__title">Apache Submarine</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a class="menu__link navbar__link--active" href="/docs/next/gettingStarted/quickstart">Docs</a></li><li class="menu__list-item"><a class="menu__link" href="/docs/next/api/environment">API</a></li><li class="menu__list-item"><a class="menu__link" href="/docs/next/download">Download</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/docs/next/designDocs/environments-implementation">master 🏃</a></li><li class="menu__list-item"><a class="menu__link" href="/docs/designDocs/environments-implementation">0.6.0</a></li><li class="menu__list-item"><a class="menu__link" href="/versions">All versions</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/apache/submarine" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li><li class="menu__list-item menu__list-item--collapsed"><a role="button" class="menu__link menu__link--sublist">Apache</a><ul class="menu__list"><li class="menu__list-item"><a href="https://www.apache.org/foundation/how-it-works.html" target="_blank" rel="noopener noreferrer" class="menu__link">Apache Software Foundation</a></li><li class="menu__list-item"><a href="https://www.apache.org/events/current-event" target="_blank" rel="noopener noreferrer" class="menu__link">Events</a></li><li class="menu__list-item"><a href="https://www.apache.org/licenses/" target="_blank" rel="noopener noreferrer" class="menu__link">Apache License</a></li><li class="menu__list-item"><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="menu__link">Thanks</a></li><li class="menu__list-item"><a href="https://www.apache.org/security/" target="_blank" rel="noopener noreferrer" class="menu__link">Security</a></li><li class="menu__list-item"><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="menu__link">Sponsorship</a></li></ul></li></ul></div></div></div></nav><div class="main-wrapper"><div class="docPage_vMrn"><div class="docSidebarContainer_3Ak5" role="complementary"><div class="sidebar_3gvy"><div class="menu menu--responsive thin-scrollbar menu_1yIk"><button aria-label="Open Menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_1CUI" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Getting Started</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/next/gettingStarted/quickstart">Quickstart</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/next/gettingStarted/notebook">Jupyter Notebook</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">User Docs</a><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!" tabindex="-1">API documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/next/userDocs/api/experiment">Experiment REST API</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/next/userDocs/api/environment">Environment REST API</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/next/userDocs/api/experiment-template">Experiment Template REST API</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/next/userDocs/api/notebook">Notebook REST API</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!" tabindex="-1">Submarine SDK</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/next/userDocs/submarine-sdk/experiment-client">Experiment Client</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/next/userDocs/submarine-sdk/submarine-cli">Submarine CLI</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/next/userDocs/submarine-sdk/tracking">Tracking</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!" tabindex="-1">Others</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/next/userDocs/others/mlflow">MLflow UI</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/next/userDocs/others/tensorboard">Tensorboard</a></li></ul></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Developer Docs</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/next/devDocs/README">Project Architecture</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/next/devDocs/Dependencies">Dependencies for Submarine</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/next/devDocs/BuildFromCode">How to Build Submarine</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/next/devDocs/Development">Development Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/next/devDocs/IntegrationTestK8s">How to Run Integration K8s Test</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/next/devDocs/IntegrationTestE2E">How to Run Frontend Integration Test</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/next/devDocs/HowToRelease">How to Release</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/next/devDocs/HowToVerify">How to Verify</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Community</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/next/community/README">Apache Submarine Community</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/next/community/Bylaws">Bylaws</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/next/community/HowToCommit">Guide for Apache Submarine Committers</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/next/community/contributing">How To Contribute to Submarine</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/next/community/HowToVoteCommitterOrPMC">How to vote a Committer or PMC</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/next/community/HowToBecomeCommitter">How to become a Committer</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Design Docs</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/docs/next/designDocs/architecture-and-requirements">Architecture and Requirment</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/docs/next/designDocs/implementation-notes">Implementation Notes</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/docs/next/designDocs/environments-implementation">Environments Implementation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/docs/next/designDocs/experiment-implementation">Experiment Implementation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/docs/next/designDocs/notebook-implementation">Notebook Implementation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/docs/next/designDocs/storage-implementation">Storage Implementation</a></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!" tabindex="0">Submarine Server</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/next/designDocs/submarine-server/architecture">Submarine Server Implementation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/next/designDocs/submarine-server/experimentSpec">Generic Experiment Spec</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!" tabindex="0">WIP Design Docs</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/next/designDocs/wip-designs/submarine-launcher">Submarine Launcher</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/next/designDocs/wip-designs/submarine-clusterServer">Cluster Server Design - High-Availability</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/next/designDocs/wip-designs/security-implementation">Security Implementation</a></li></ul></li></ul></li></ul></div></div></div><main class="docMainContainer_2iGs"><div class="container padding-vert--lg docItemWrapper_1bxp"><div class="row"><div class="col docItemCol_U38p"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Apache Submarine <strong>master 🏃</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/docs/designDocs/environments-implementation">latest version</a></strong> (0.6.0).</div></div><div class="docItemContainer_a7m4"><article><div><span class="badge badge--secondary">Version: master 🏃</span></div><header><h1 class="docTitle_Oumm">Environments Implementation</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_prK2" id="overview"></a>Overview<a class="hash-link" href="#overview" title="Direct link to heading">#</a></h2><p>Environment profiles (or environment for short) defines a set of libraries and when Docker is being used, a Docker image in order to run an experiment or a notebook. </p><p>Docker and/or VM-image (such as, VirtualBox/VMWare images, Amazon Machine Images - AMI, Or custom image of Azure VM) defines the base layer of the environment. Please note that VM-image is different from VM instance type,</p><p>On top of that, users can define a set of libraries (such as Python/R) to install, we call it kernel.</p><p><strong>Example of Environment</strong></p><div class="mdxCodeBlock_1zKU"><div class="codeBlockContent_actS"><div tabindex="0" class="prism-code language-undefined codeBlock_tuNs thin-scrollbar"><div class="codeBlockLines_3uvA" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"> |
| </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> +-------------------+</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> |+-----------------+|</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> || Python=3.7 ||</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> || Tensorflow=2.0 ||</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> |+---Exp Dependency+|</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> |+-----------------+|</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> ||OS=Ubuntu16.04 ||</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> ||CUDA=10.2 ||</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> ||GPU_Driver=375.. ||</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> |+---Base Library--+|</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> +-------------------+</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_2GIj">Copy</button></div></div><p>As you can see, There're base libraries, such as what OS, CUDA version, GPU driver, etc. They can be achieved by specifying a VM-image / Docker image.</p><p>On top of that, user can bring their dependencies, such as different version of Python, Tensorflow, Pandas, etc.</p><p><strong>How users use environment?</strong></p><p>Users can save different environment configs which can be also shared across the platform. Environment profiles can be used to run a notebook (e.g. by choosing different kernel from Jupyter), or an experiment. Predefined experiment library includes what environment to use so users don't have to choose which environment to use.</p><div class="mdxCodeBlock_1zKU"><div class="codeBlockContent_actS"><div tabindex="0" class="prism-code language-undefined codeBlock_tuNs thin-scrollbar"><div class="codeBlockLines_3uvA" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"> |
| </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> +-------------------+</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> |+-----------------+| +------------+</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> || Python=3.7 || |User1 |</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> || Tensorflow=2.0 || +------------+</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> |+---Kernel -------+| +------------+</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> |+-----------------+|<----+ |User2 |</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> ||OS=Ubuntu16.04 || + +------------+</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> ||CUDA=10.2 || | +------------+</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> ||GPU_Driver=375.. || | |User3 |</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> |+---Base Library--+| | +------------+</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> +-----Default-Env---+ |</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> |</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> |</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> +-------------------+ |</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> |+-----------------+| |</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> || Python=3.3 || |</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> || Tensorflow=2.0 || |</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> |+---kernel--------+| |</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> |+-----------------+| |</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> ||OS=Ubuntu16.04 || |</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> ||CUDA=10.3 ||<----+</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> ||GPU_Driver=375.. ||</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> |+---Base Library--+|</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> +-----My-Customized-+</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_2GIj">Copy</button></div></div><p>There're two environments in the above graph, "Default-Env" and "My-Customized", which can have different combinations of libraries for different experiments/notebooks. Users can choose different environments for different experiments as they want.</p><p>Environments can be added/listed/deleted/selected through CLI/SDK/UI.</p><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_prK2" id="implementation"></a>Implementation<a class="hash-link" href="#implementation" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_prK2" id="environment-api-definition"></a>Environment API definition<a class="hash-link" href="#environment-api-definition" title="Direct link to heading">#</a></h2><p>Let look at what object definition looks like to define an environment, API of environment looks like:</p><div class="mdxCodeBlock_1zKU"><div class="codeBlockContent_actS"><div tabindex="0" class="prism-code language-undefined codeBlock_tuNs thin-scrollbar"><div class="codeBlockLines_3uvA" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"> name: "my_submarine_env",</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> vm-image: "...",</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> docker-image: "...", </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> kernel: </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> <object of kernel></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> description: "this is the most common env used by team ABC"</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_2GIj">Copy</button></div></div><ul><li><code>vm-image</code> is optional if we don't need to launch new VM (like running a training job in a cloud-remote machine). </li><li><code>docker-image</code> is required</li><li><code>kernel</code> could be optional if kernel is already included by vm-image or docker-image.</li><li><code>name</code> of the environment should be unique in the system, so user can reference it when create a new experiment/notebook.</li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_prK2" id="vm-image-and-docker-image"></a>VM-image and Docker-image<a class="hash-link" href="#vm-image-and-docker-image" title="Direct link to heading">#</a></h2><p>Docker-image and VM image should be prepared by system admin / SREs, it is hard for Data-Scientists to write an error-proof Dockerfile, and push/manage Docker images. This is one of the reason we hide Docker-image inside "environment", we will encourage users to customize their kernels if needed, but don't have to touch Dockerfile and build/push/manage new Docker images.</p><p>As a project, we will document what's the best practice and example of Dockerfiles. </p><p>Dockerfile should include proper <code>ENTRYPOINT</code> definition which pointed to our default script, so no matter it is notebook, or an experiment, we will setup kernel (see below) and other environment variables properly.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_prK2" id="kernel-implementation"></a>Kernel Implementation<a class="hash-link" href="#kernel-implementation" title="Direct link to heading">#</a></h2><p>After investigating different alternatives (such as pipenv, venv, etc.), we decided to use Conda environment which nicely replaces Python virtual env, pip, and can also support other languages. More details can be found at: <a href="https://medium.com/@krishnaregmi/pipenv-vs-virtualenv-vs-conda-environment-3dde3f6869ed" target="_blank" rel="noopener noreferrer">https://medium.com/@krishnaregmi/pipenv-vs-virtualenv-vs-conda-environment-3dde3f6869ed</a></p><p>When once Conda, users can easily add, remove dependency of a Conda environment. User can also easily export environment to yaml file.</p><p>The yaml file of Conda environment by using <code>conda env export</code> looks like: </p><div class="mdxCodeBlock_1zKU"><div class="codeBlockContent_actS"><div tabindex="0" class="prism-code language-undefined codeBlock_tuNs thin-scrollbar"><div class="codeBlockLines_3uvA" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">name: base</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">channels:</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> - defaults</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">dependencies:</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> - _ipyw_jlab_nb_ext_conf=0.1.0=py37_0</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> - alabaster=0.7.12=py37_0</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> - anaconda=2020.02=py37_0</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> - anaconda-client=1.7.2=py37_0</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> - anaconda-navigator=1.9.12=py37_0</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> - anaconda-project=0.8.4=py_0</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> - applaunchservices=0.2.1=py_0</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_2GIj">Copy</button></div></div><p>Including Conda kernel, the environment object may look like: </p><div class="mdxCodeBlock_1zKU"><div class="codeBlockContent_actS"><div tabindex="0" class="prism-code language-undefined codeBlock_tuNs thin-scrollbar"><div class="codeBlockLines_3uvA" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">name: "my_submarine_env",</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> vm-image: "...",</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> docker-image: "...", </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> kernel: </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> name: team_default_python_3.7</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> channels:</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> - defaults</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> dependencies:</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> - _ipyw_jlab_nb_ext_conf=0.1.0=py37_0</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> - alabaster=0.7.12=py37_0</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> - anaconda=2020.02=py37_0</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> - anaconda-client=1.7.2=py37_0</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> - anaconda-navigator=1.9.12=py37_0</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_2GIj">Copy</button></div></div><p>When launch a new experiment / notebook session using the <code>my_submarine_env</code>, submarine server will use defined Docker image, and Conda kernel to launch of container. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_prK2" id="storage-of-environment"></a>Storage of Environment<a class="hash-link" href="#storage-of-environment" title="Direct link to heading">#</a></h2><p>Environment of Submarine is just a simple text file, so it will be persisted in Submarine metastore, which is ideally a Database. </p><p>Docker image is stored inside a regular Docker registry, which will be handled outside of the system. </p><p>Conda dependencies are stored in Conda channel (where referenced packages are stored), which will be handled/setuped separately. (Popular conda channels are <code>default</code> and <code>conda-forge</code>)</p><p>For more detailed discussion about storage-related implementations, please refer to <a href="/docs/next/designDocs/storage-implementation">storage-implementation</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_prK2" id="how-to-implement-to-make-user-can-easily-use-submarine-environments"></a>How to implement to make user can easily use Submarine environments?<a class="hash-link" href="#how-to-implement-to-make-user-can-easily-use-submarine-environments" title="Direct link to heading">#</a></h2><p>We like simplicities, and we don't want to leak complexities of implementations to the users. To make it happen, we have to do some works to hide complexities. </p><p>There're two primary uses of environments: experiments and notebook, for both of them, users should not do works like explictily call <code>conda active $env_name</code> to active environments. To make it happen, what we can do is to include following parts in Dockerfile </p><div class="mdxCodeBlock_1zKU"><div class="codeBlockContent_actS"><div tabindex="0" class="prism-code language-undefined codeBlock_tuNs thin-scrollbar"><div class="codeBlockLines_3uvA" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">FROM ubuntu:18.04</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"> |
| </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"><Include whatever base-libraries like CUDA, etc.></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"> |
| </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"><Make sure conda (with our preferred version) is installed></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"><Make sure Jupyter (with our preferred version) is installed></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"> |
| </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"># This is just a sample of Dockerfile, users can do more customizations if needed</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ENTRYPOINT ["/submarine-bootstrap.sh"]</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_2GIj">Copy</button></div></div><p>When Submarine Server (this is implementation detail of Submarine Server, user will not see it at all) launch an experiment, or notebook, it will invoke following <code>docker run</code> command (or any other equvilant like using K8s spec): </p><div class="mdxCodeBlock_1zKU"><div class="codeBlockContent_actS"><div tabindex="0" class="prism-code language-undefined codeBlock_tuNs thin-scrollbar"><div class="codeBlockLines_3uvA" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker run <submarine_docker_image> --kernel <kernel_name> -- .... python train.py --batch_size 5 (and other parameters)</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_2GIj">Copy</button></div></div><p>Similarily, to launch a notebook: </p><div class="mdxCodeBlock_1zKU"><div class="codeBlockContent_actS"><div tabindex="0" class="prism-code language-undefined codeBlock_tuNs thin-scrollbar"><div class="codeBlockLines_3uvA" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker run <submarine_docker_image> --kernel <kernel_name> -- .... jupyter</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_2GIj">Copy</button></div></div><p>The <code>submarine-bootstrap.sh</code> is part of Submarine repo, and will handle <code>--kernel</code> argument which will invoke <code>conda active $kernel_name</code> before anything else. (Like run the training job).</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/apache/submarine/edit/master/website/docs/designDocs/environments-implementation.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2LL7"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Blog list page navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/docs/next/designDocs/implementation-notes"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Implementation Notes</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/docs/next/designDocs/experiment-implementation"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Experiment Implementation »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_2xL- thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#overview" class="table-of-contents__link">Overview</a></li><li><a href="#environment-api-definition" class="table-of-contents__link">Environment API definition</a></li><li><a href="#vm-image-and-docker-image" class="table-of-contents__link">VM-image and Docker-image</a></li><li><a href="#kernel-implementation" class="table-of-contents__link">Kernel Implementation</a></li><li><a href="#storage-of-environment" class="table-of-contents__link">Storage of Environment</a></li><li><a href="#how-to-implement-to-make-user-can-easily-use-submarine-environments" class="table-of-contents__link">How to implement to make user can easily use Submarine environments?</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="row footer__links"><div class="col footer__col"><h4 class="footer__title">Docs</h4><ul class="footer__items"><li class="footer__item"><a class="footer__link-item" href="/docs/gettingStarted/quickstart">Getting Started</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/api/environment">API docs</a></li></ul></div><div class="col footer__col"><h4 class="footer__title">Community</h4><ul class="footer__items"><li class="footer__item"><a href="https://stackoverflow.com/questions/tagged/apache-submarine" target="_blank" rel="noopener noreferrer" class="footer__link-item">Stack Overflow</a></li><li class="footer__item"><a href="https://s.apache.org/slack-invite" target="_blank" rel="noopener noreferrer" class="footer__link-item">Slack</a></li></ul></div><div class="col footer__col"><h4 class="footer__title">More</h4><ul class="footer__items"><li class="footer__item"><a href="https://medium.com/@apache.submarine" target="_blank" rel="noopener noreferrer" class="footer__link-item">Blog</a></li><li class="footer__item"><a href="https://github.com/apache/submarine" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub</a></li></ul></div></div><div class="footer__bottom text--center"><div class="margin-bottom--sm"><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer" class="footerLogoLink_31Aa"><img class="footer__logo" alt="Apache Open Source Logo" src="https://hadoop.apache.org/asf_logo_wide.png"></a></div><div class="footer__copyright">Apache Submarine, Submarine, Apache, the Apache feather logo, and the Apache Submarine project logo are |
| either registered trademarks or trademarks of the Apache Software Foundation in the United States and other |
| countries.<br> Copyright © 2022 Apache Submarine is Apache2 Licensed software.</div></div></div></footer></div> |
| <script src="/styles.6ddab7ad.js"></script> |
| <script src="/runtime~main.943b9f3a.js"></script> |
| <script src="/main.92db4fb0.js"></script> |
| <script src="/1.dab80c77.js"></script> |
| <script src="/2.6a1a8326.js"></script> |
| <script src="/1f391b9e.f22e8160.js"></script> |
| <script src="/128.10c43f42.js"></script> |
| <script src="/935f2afb.9989d552.js"></script> |
| <script src="/17896441.be8f95bf.js"></script> |
| <script src="/a7e506f8.92088415.js"></script> |
| </body> |
| </html> |
