| <!doctype html> |
| <html lang="en"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="viewport" content="width=device-width,initial-scale=1"> |
| <meta name="generator" content="Docusaurus v2.0.0-alpha.70"> |
| <link rel="alternate" type="application/rss+xml" href="/blog/rss.xml" title="Apache Submarine Blog RSS Feed"> |
| <link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Submarine Blog Atom Feed"><title data-react-helmet="true">Storage Implementation | Apache Submarine</title><meta data-react-helmet="true" name="twitter:card" content="summary_large_image"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="0.6.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-0.6.0"><meta data-react-helmet="true" property="og:title" content="Storage Implementation | Apache Submarine"><meta data-react-helmet="true" name="description" content="<!--"><meta data-react-helmet="true" property="og:description" content="<!--"><meta data-react-helmet="true" property="og:url" content="https://submarine.apache.org//docs/designDocs/storage-implementation"><link data-react-helmet="true" rel="shortcut icon" href="/img/submarine.ico"><link data-react-helmet="true" rel="canonical" href="https://submarine.apache.org//docs/designDocs/storage-implementation"><link rel="stylesheet" href="/styles.39775f96.css"> |
| <link rel="preload" href="/styles.f6b0c2f2.js" as="script"> |
| <link rel="preload" href="/runtime~main.13a9404d.js" as="script"> |
| <link rel="preload" href="/main.1c145c17.js" as="script"> |
| <link rel="preload" href="/1.d23d1451.js" as="script"> |
| <link rel="preload" href="/2.45bcb8a0.js" as="script"> |
| <link rel="preload" href="/1f391b9e.785b37ba.js" as="script"> |
| <link rel="preload" href="/127.875bba76.js" as="script"> |
| <link rel="preload" href="/58f10d9f.e974ccf6.js" as="script"> |
| <link rel="preload" href="/17896441.faf04472.js" as="script"> |
| <link rel="preload" href="/538541c1.2e7721f8.js" as="script"> |
| </head> |
| <body> |
| <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus"> |
| <nav aria-label="Skip navigation links"><button type="button" tabindex="0" class="skipToContent_11B0">Skip to main content</button></nav><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><div aria-label="Navigation bar toggle" class="navbar__toggle" role="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></div><a class="navbar__brand" href="/"><img src="/img/icons/128.png" alt="Apache Submarine Site Logo" class="themedImage_YANc themedImage--light_3CMI navbar__logo"><img src="/img/icons/128.png" alt="Apache Submarine Site Logo" class="themedImage_YANc themedImage--dark_3ARp navbar__logo"><strong class="navbar__title">Apache Submarine</strong></a><a class="navbar__item navbar__link navbar__link--active" href="/docs/gettingStarted/quickstart">Docs</a><a class="navbar__item navbar__link" href="/docs/api/environment">API</a><a class="navbar__item navbar__link" href="/docs/download">Download</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/docs/gettingStarted/quickstart">0.6.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/docs/next/designDocs/storage-implementation">master 🏃</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/docs/designDocs/storage-implementation">0.6.0</a></li><li><a class="dropdown__link" href="/versions">All versions</a></li></ul></div><a href="https://github.com/apache/submarine" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link">Apache</a><ul class="dropdown__menu"><li><a href="https://www.apache.org/foundation/how-it-works.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Apache Software Foundation</a></li><li><a href="https://www.apache.org/events/current-event" target="_blank" rel="noopener noreferrer" class="dropdown__link">Events</a></li><li><a href="https://www.apache.org/licenses/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Apache License</a></li><li><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Thanks</a></li><li><a href="https://www.apache.org/security/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Security</a></li><li><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Sponsorship</a></li></ul></div><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_2N3Q"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_3NWk">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_3NWk">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div><div class="navbar__search"><span aria-label="expand searchbar" role="button" class="search-icon" tabindex="0"></span><input type="search" id="search_input_react" placeholder="Search" aria-label="Search" class="navbar__search-input search-bar"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/"><img src="/img/icons/128.png" alt="Apache Submarine Site Logo" class="themedImage_YANc themedImage--light_3CMI navbar__logo"><img src="/img/icons/128.png" alt="Apache Submarine Site Logo" class="themedImage_YANc themedImage--dark_3ARp navbar__logo"><strong class="navbar__title">Apache Submarine</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a class="menu__link navbar__link--active" href="/docs/gettingStarted/quickstart">Docs</a></li><li class="menu__list-item"><a class="menu__link" href="/docs/api/environment">API</a></li><li class="menu__list-item"><a class="menu__link" href="/docs/download">Download</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/docs/next/designDocs/storage-implementation">master 🏃</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/docs/designDocs/storage-implementation">0.6.0</a></li><li class="menu__list-item"><a class="menu__link" href="/versions">All versions</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/apache/submarine" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li><li class="menu__list-item menu__list-item--collapsed"><a role="button" class="menu__link menu__link--sublist">Apache</a><ul class="menu__list"><li class="menu__list-item"><a href="https://www.apache.org/foundation/how-it-works.html" target="_blank" rel="noopener noreferrer" class="menu__link">Apache Software Foundation</a></li><li class="menu__list-item"><a href="https://www.apache.org/events/current-event" target="_blank" rel="noopener noreferrer" class="menu__link">Events</a></li><li class="menu__list-item"><a href="https://www.apache.org/licenses/" target="_blank" rel="noopener noreferrer" class="menu__link">Apache License</a></li><li class="menu__list-item"><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="menu__link">Thanks</a></li><li class="menu__list-item"><a href="https://www.apache.org/security/" target="_blank" rel="noopener noreferrer" class="menu__link">Security</a></li><li class="menu__list-item"><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="menu__link">Sponsorship</a></li></ul></li></ul></div></div></div></nav><div class="main-wrapper"><div class="docPage_vMrn"><div class="docSidebarContainer_3Ak5" role="complementary"><div class="sidebar_3gvy"><div class="menu menu--responsive thin-scrollbar menu_1yIk"><button aria-label="Open Menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_1CUI" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Getting Started</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/gettingStarted/quickstart">Quickstart</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/gettingStarted/notebook">Jupyter Notebook</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">User Docs</a><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!" tabindex="-1">API documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/userDocs/api/experiment">Experiment REST API</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/userDocs/api/environment">Environment REST API</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/userDocs/api/experiment-template">Experiment Template REST API</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/userDocs/api/notebook">Notebook REST API</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!" tabindex="-1">Submarine SDK</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/userDocs/submarine-sdk/experiment-client">Experiment Client</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/userDocs/submarine-sdk/model-client">Model Client</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/userDocs/submarine-sdk/tracking">Tracking</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!" tabindex="-1">Submarine Security</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/userDocs/submarine-security/spark-security/README">Submarine Spark Security Plugin</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/userDocs/submarine-security/spark-security/build-submarine-spark-security-plugin">Building Submarine Spark Security Plugin</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!" tabindex="-1">Others</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/userDocs/others/mlflow">MLflow UI</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/userDocs/others/tensorboard">Tensorboard</a></li></ul></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Administrator Docs</a><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!" tabindex="-1">Submarine on Yarn</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/adminDocs/yarn/README">Running Submarine on YARN</a></li></ul></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Developer Docs</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/devDocs/README">Project Architecture</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/devDocs/Dependencies">Dependencies for Submarine</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/devDocs/BuildFromCode">How to Build Submarine</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/devDocs/Development">Development Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/devDocs/IntegrationTestK8s">How to Run Integration K8s Test</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/devDocs/IntegrationTestE2E">How to Run Frontend Integration Test</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Community</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/community/README">Apache Submarine Community</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/community/HowToCommit">Guide for Apache Submarine Committers</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/community/contributing">How To Contribute to Submarine</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Design Docs</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/docs/designDocs/architecture-and-requirements">Architecture and Requirment</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/docs/designDocs/implementation-notes">Implementation Notes</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/docs/designDocs/environments-implementation">Environments Implementation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/docs/designDocs/experiment-implementation">Experiment Implementation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/docs/designDocs/notebook-implementation">Notebook Implementation</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/docs/designDocs/storage-implementation">Storage Implementation</a></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!" tabindex="0">Submarine Server</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/designDocs/submarine-server/architecture">Submarine Server Implementation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/designDocs/submarine-server/experimentSpec">Generic Expeiment Spec</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!" tabindex="0">WIP Design Docs</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/designDocs/wip-designs/submarine-launcher">Submarine Launcher</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/designDocs/wip-designs/submarine-clusterServer">Cluster Server Design - High-Availability</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/docs/designDocs/wip-designs/security-implementation">Security Implementation</a></li></ul></li></ul></li></ul></div></div></div><main class="docMainContainer_2iGs"><div class="container padding-vert--lg docItemWrapper_1bxp"><div class="row"><div class="col docItemCol_U38p"><div class="docItemContainer_a7m4"><article><div><span class="badge badge--secondary">Version: 0.6.0</span></div><header><h1 class="docTitle_Oumm">Storage Implementation</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_prK2" id="ml-related-objects-and-their-storages"></a>ML-related objects and their storages<a class="hash-link" href="#ml-related-objects-and-their-storages" title="Direct link to heading">#</a></h2><p>First let's look at what user will interact for most of the time: </p><ul><li>Notebook </li><li>Experiment</li><li>Model Servings</li></ul><div class="mdxCodeBlock_1zKU"><div class="codeBlockContent_actS"><div tabindex="0" class="prism-code language-undefined codeBlock_tuNs thin-scrollbar"><div class="codeBlockLines_3uvA" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"> |
| </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"> |
| </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> +---------+ +------------+</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> |Logs |<--+|Notebook |</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> +----------+ +---------+ +------------+ +----------------+</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> |Trackings | <-+|Experiment |<--+>|Model Artifacts |</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> +----------+ +-----------------+ +------------+ +----------------+</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> +----------+<---+|ML-related Metric|<--+Servings |</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> |tf.events | +-----------------+ +------------+</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> +----------+ ^ +-----------------+</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> + | Environments |</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> +----------------------+ | |</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> +-----------------+ | Submarine Metastore | | Dependencies |</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> |Code | +----------------------+ | |</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> +-----------------+ |Experiment Meta | | Docker Images |</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> +----------------------+ +-----------------+</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> |Model Store Meta |</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> +----------------------+</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> |Model Serving Meta |</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> +----------------------+</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> |Notebook meta |</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> +----------------------+</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> |Experiment Templates |</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> +----------------------+</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> |Environments Meta |</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> +----------------------+</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_2GIj">Copy</button></div></div><p>First of all, all the notebook-sessions / experiments / model-serving instances) are more or less interact with following storage objects:</p><ul><li>Logs for these tasks for troubleshooting. </li><li>ML-related metrics such as loss, epoch, etc. (in contrast of system metrics such as CPU/memory usage, etc.)<ul><li>There're different types of ML-related metrics, for Tensorflow/pytorch, they can use tf.events and get visualizations on tensorboard. </li><li>Or they can use tracking APIs (such as Submarine tracking, mlflow tracking, etc.) to output customized tracking results for non TF/Pytorch workloads. </li></ul></li><li>Training jobs of experiment typically generate model artifacts (files) which need persisted, and both of notebook, model serving needs to load model artifacts from persistent storage. </li><li>There're various of meta information, such as experiment meta, model registry, model serving, notebook, experiment, environment, etc. We need be able to read these meta information back.</li><li>We also have code for experiment (like training/batch-prediction), notebook (ipynb), and model servings.</li><li>And notebook/experiments/model-serving need depend on environments (dependencies such as pip, and Docker Images).</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_prK2" id="implementation-considerations-for-ml-related-objects"></a>Implementation considerations for ML-related objects<a class="hash-link" href="#implementation-considerations-for-ml-related-objects" title="Direct link to heading">#</a></h3><table><thead><tr><th>Object Type</th><th>Characteristics</th><th>Where to store</th></tr></thead><tbody><tr><td>Metrics: tf.events</td><td>Time series data with k/v, appendable to file</td><td>Local/EBS, HDFS, Cloud Blob Storage</td></tr><tr><td>Metrics: other tracking metrics</td><td>Time series data with k/v, appendable to file</td><td>Local, HDFS, Cloud Blob Storage, Database</td></tr><tr><td>Logs</td><td>Large volumes, #files are potentially huge.</td><td>Local (temporary), HDFS (need aggregation), Cloud Blob Storage</td></tr><tr><td>Submarine Metastore</td><td>CRUD operations for small meta data.</td><td>Database</td></tr><tr><td>Model Artifacts</td><td>Size varies for model (from KBs to GBs). #files are potentially huge.</td><td>HDFS, Cloud Blob Storage</td></tr><tr><td>Code</td><td>Need version control. (Please find detailed discussions below for code storage and localization)</td><td>Tarball on HDFS/Cloud Blog Storage, or Git</td></tr><tr><td>Environment (Dependencies, Docker Image)</td><td></td><td>Public/private environment repo (like Conda channel), Docker registry.</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_prK2" id="detailed-discussions"></a>Detailed discussions<a class="hash-link" href="#detailed-discussions" title="Direct link to heading">#</a></h3><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_prK2" id="store-code-for-experimentnotebookmodel-serving"></a>Store code for experiment/notebook/model-serving<a class="hash-link" href="#store-code-for-experimentnotebookmodel-serving" title="Direct link to heading">#</a></h4><p>There're following ways to get experiment code: </p><p><strong>1) Code is part of Git repo:</strong> (<strong><em><u>Recommended</u></em></strong>)</p><p>This is our recommended approach, once code is part of Git, it will be stored in version control, any change will be tracked, and much easier for users to trace back what change triggered a new bug, etc.</p><p><strong>2) Code is part of Docker image:</strong> </p><p><strong><em>This is an anti-pattern and we will NOT recommend you to use it</em></strong>, Docker image can be used to include ANYTHING, like dependencies, the code you will execute, or even data. But this doesn't mean you should do it. We recommend to use Docker image ONLY for libraries/dependencies.</p><p>Making code to be part of Docker image makes hard to edit code (if you want to update a value in your Python file, you will have to recreate the Docker image, push it and rerun it).</p><p><strong>3) Code is part of S3/HDFS/ABFS:</strong> </p><p>User may want to store their training code to a tarball on a shared storage. Submarine need to download code from remote storage to the launched container before running the code. </p><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_prK2" id="localization-of-experimentnotebookmodel-serving-code"></a>Localization of experiment/notebook/model-serving code<a class="hash-link" href="#localization-of-experimentnotebookmodel-serving-code" title="Direct link to heading">#</a></h4><p>To make user experiences keeps same across different environment, we will localize code to a same folder after the container is launched, preferably <code>/code</code></p><p>For example, there's a git repo need to be synced up for an experiment/notebook/model-serving (example above):</p><div class="mdxCodeBlock_1zKU"><div class="codeBlockContent_actS"><div tabindex="0" class="prism-code language-undefined codeBlock_tuNs thin-scrollbar"><div class="codeBlockLines_3uvA" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">experiment: #Or notebook, model-serving</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> name: "abc",</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> environment: "team-default-ml-env"</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> ... (other fields)</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> code:</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> sync_mode: git</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> url: "https://foo.com/training-job.git" </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_2GIj">Copy</button></div></div><p>After localize, <code>training-job/</code> will be placed under <code>/code</code> </p><p>When we running on K8s environment, we can use K8s's initContainer and emptyDir to do these things for us. K8s POD spec (generated by Submarine server instead of user, user should NEVER edit K8s spec, that's too unfriendly to data-scientists): </p><div class="mdxCodeBlock_1zKU"><div class="codeBlockContent_actS"><div tabindex="0" class="prism-code language-undefined codeBlock_tuNs thin-scrollbar"><div class="codeBlockLines_3uvA" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">apiVersion: v1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">kind: Pod</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">metadata:</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> name: experiment-abc</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">spec:</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> containers:</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> - name: experiment-task</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> image: training-job</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> volumeMounts:</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> - name: code-dir</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> mountPath: /code</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> initContainers:</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> - name: git-localize</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> image: git-sync</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> command: "git clone .. /code/"</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> volumeMounts:</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> - name: code-dir</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> mountPath: /code</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> volumes:</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> - name: code-dir</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> emptyDir: {}</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_2GIj">Copy</button></div></div><p>The above K8s spec create a code-dir and mount it to <code>/code</code> to launched containers. The initContainer <code>git-localize</code> uses <code>https://github.com/kubernetes/git-sync</code> to do the sync up. (If other storages are used such as s3, we can use similar initContainer approach to download contents)</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_prK2" id="system-related-metricslogs-and-their-storages"></a>System-related metrics/logs and their storages<a class="hash-link" href="#system-related-metricslogs-and-their-storages" title="Direct link to heading">#</a></h2><p>Other than ML-related objects, we have system-related objects, including: </p><ul><li>Daemon logs (like logs of Submarine server). </li><li>Logs for other dependency components (like Kubernetes logs when running on K8s). </li><li>System metrics (Physical resource usages by daemons, launched training containers, etc.). </li></ul><p>All these information should be handled by 3rd party system, such as Grafana, Prometheus, etc. And system admins are responsible to setup these infrastructures, dashboard. Users of submarine should NOT interact with system related metrics/logs. It is system admin's responsibility.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_prK2" id="attachable-volumes"></a>Attachable Volumes<a class="hash-link" href="#attachable-volumes" title="Direct link to heading">#</a></h2><p>It is possible user has needs to have an attachable volume for their experiment / notebook, this is especially useful for notebook storage, since contents of notebook can be automatically saved, and it can be used as user's home folder. </p><p>Downside of attachable volume is, it is not versioned, even notebook is mainly used for adhoc exploring tasks, an unversioned notebook file can lead to maintenance issues in the future. </p><p>Since this is a common requirement, we can consider to support attachable volumes in Submarine in a long run, but with relatively lower priority.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_prK2" id="in-scope--out-of-scope"></a>In-scope / Out-of-scope<a class="hash-link" href="#in-scope--out-of-scope" title="Direct link to heading">#</a></h2><p> Describe what Submarine project should own and what Submarine project should NOT own.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/apache/submarine/edit/master/website/versioned_docs/version-0.6.0/designDocs/storage-implementation.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2LL7"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Blog list page navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/docs/designDocs/notebook-implementation"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Notebook Implementation</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/docs/designDocs/submarine-server/architecture"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Submarine Server Implementation »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_2xL- thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#ml-related-objects-and-their-storages" class="table-of-contents__link">ML-related objects and their storages</a><ul><li><a href="#implementation-considerations-for-ml-related-objects" class="table-of-contents__link">Implementation considerations for ML-related objects</a></li><li><a href="#detailed-discussions" class="table-of-contents__link">Detailed discussions</a></li></ul></li><li><a href="#system-related-metricslogs-and-their-storages" class="table-of-contents__link">System-related metrics/logs and their storages</a></li><li><a href="#attachable-volumes" class="table-of-contents__link">Attachable Volumes</a></li><li><a href="#in-scope--out-of-scope" class="table-of-contents__link">In-scope / Out-of-scope</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="row footer__links"><div class="col footer__col"><h4 class="footer__title">Docs</h4><ul class="footer__items"><li class="footer__item"><a class="footer__link-item" href="/docs/gettingStarted/quickstart">Getting Started</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/api/environment">API docs</a></li></ul></div><div class="col footer__col"><h4 class="footer__title">Community</h4><ul class="footer__items"><li class="footer__item"><a href="https://stackoverflow.com/questions/tagged/apache-submarine" target="_blank" rel="noopener noreferrer" class="footer__link-item">Stack Overflow</a></li><li class="footer__item"><a href="https://s.apache.org/slack-invite" target="_blank" rel="noopener noreferrer" class="footer__link-item">Slack</a></li></ul></div><div class="col footer__col"><h4 class="footer__title">More</h4><ul class="footer__items"><li class="footer__item"><a href="https://medium.com/@apache.submarine" target="_blank" rel="noopener noreferrer" class="footer__link-item">Blog</a></li><li class="footer__item"><a href="https://github.com/apache/submarine" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub</a></li></ul></div></div><div class="footer__bottom text--center"><div class="margin-bottom--sm"><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer" class="footerLogoLink_31Aa"><img class="footer__logo" alt="Apache Open Source Logo" src="https://hadoop.apache.org/asf_logo_wide.png"></a></div><div class="footer__copyright">Apache Submarine, Submarine, Apache, the Apache feather logo, and the Apache Submarine project logo are |
| either registered trademarks or trademarks of the Apache Software Foundation in the United States and other |
| countries.<br> Copyright © 2022 Apache Submarine is Apache2 Licensed software.</div></div></div></footer></div> |
| <script src="/styles.f6b0c2f2.js"></script> |
| <script src="/runtime~main.13a9404d.js"></script> |
| <script src="/main.1c145c17.js"></script> |
| <script src="/1.d23d1451.js"></script> |
| <script src="/2.45bcb8a0.js"></script> |
| <script src="/1f391b9e.785b37ba.js"></script> |
| <script src="/127.875bba76.js"></script> |
| <script src="/58f10d9f.e974ccf6.js"></script> |
| <script src="/17896441.faf04472.js"></script> |
| <script src="/538541c1.2e7721f8.js"></script> |
| </body> |
| </html> |
