<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="x-ua-compatible" content="ie=edge"/><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"/><meta name="generator" content="Gatsby 2.25.0"/><style type="text/css">
    .anchor.before {
      position: absolute;
      top: 0;
      left: 0;
      transform: translateX(-100%);
      padding-right: 4px;
    }
    .anchor.after {
      display: inline-block;
      padding-left: 4px;
    }
    h1 .anchor svg,
    h2 .anchor svg,
    h3 .anchor svg,
    h4 .anchor svg,
    h5 .anchor svg,
    h6 .anchor svg {
      visibility: hidden;
    }
    h1:hover .anchor svg,
    h2:hover .anchor svg,
    h3:hover .anchor svg,
    h4:hover .anchor svg,
    h5:hover .anchor svg,
    h6:hover .anchor svg,
    h1 .anchor:focus svg,
    h2 .anchor:focus svg,
    h3 .anchor:focus svg,
    h4 .anchor:focus svg,
    h5 .anchor:focus svg,
    h6 .anchor:focus svg {
      visibility: visible;
    }
  </style><script>
    document.addEventListener("DOMContentLoaded", function(event) {
      var hash = window.decodeURI(location.hash.replace('#', ''))
      if (hash !== '') {
        var element = document.getElementById(hash)
        if (element) {
          var scrollTop = window.pageYOffset || document.documentElement.scrollTop || document.body.scrollTop
          var clientTop = document.documentElement.clientTop || document.body.clientTop || 0
          var offset = element.getBoundingClientRect().top + scrollTop - clientTop
          // Wait for the browser to finish rendering before scrolling.
          setTimeout((function() {
            window.scrollTo(0, offset - 0)
          }), 0)
        }
      }
    })
  </script><title data-react-helmet="true">Liminal</title><meta data-react-helmet="true" name="description" content="Apache Limial official site"/><meta data-react-helmet="true" name="image" content="http://liminal.apache.com/logo.png"/><meta data-react-helmet="true" http-equiv="x-ua-compatible" content="IE=edge,chrome=1"/><meta data-react-helmet="true" name="MobileOptimized" content="320"/><meta data-react-helmet="true" name="HandheldFriendly" content="True"/><meta data-react-helmet="true" name="google" content="notranslate"/><meta data-react-helmet="true" name="referrer" content="no-referrer-when-downgrade"/><meta data-react-helmet="true" property="og:url" content="http://liminal.apache.com/usage/arch/"/><meta data-react-helmet="true" property="og:type" content="website"/><meta data-react-helmet="true" property="og:title" content="Liminal"/><meta data-react-helmet="true" property="og:description" content="Apache Limial official site"/><meta data-react-helmet="true" property="og:locale" content="en"/><meta data-react-helmet="true" property="og:site_name" content=""/><meta data-react-helmet="true" property="og:image" content="http://liminal.apache.com/logo.png"/><meta data-react-helmet="true" property="og:image:secure_url" content="http://liminal.apache.com/logo.png"/><meta data-react-helmet="true" property="og:image:alt" content="Banner"/><meta data-react-helmet="true" property="og:image:type" content="image/png"/><meta data-react-helmet="true" property="og:image:width" content="1200"/><meta data-react-helmet="true" property="og:image:height" content="630"/><meta data-react-helmet="true" name="twitter:card" content="summary_large_image"/><meta data-react-helmet="true" name="twitter:title" content="Liminal"/><meta data-react-helmet="true" name="twitter:site" content="@lior.schachter"/><meta data-react-helmet="true" name="twitter:creator" content="@lior.schachter"/><meta data-react-helmet="true" name="twitter:image" content="http://liminal.apache.com/logo.png"/><meta data-react-helmet="true" name="twitter:image:src" content="http://liminal.apache.com/logo.png"/><meta data-react-helmet="true" name="twitter:image:alt" content="Banner"/><meta data-react-helmet="true" name="twitter:image:width" content="1200"/><meta data-react-helmet="true" name="twitter:image:height" content="630"/><script data-react-helmet="true" type="application/ld+json">[{"@context":"http://schema.org","@type":"WebSite","url":"http://liminal.apache.com/usage/arch/","name":"","alternateName":"Apache Liminal"}]</script><link rel="icon" href="/favicon-32x32.png?v=f730a0b8e533f2d4da7bd901d2b7ab80" type="image/png"/><link rel="manifest" href="/manifest.webmanifest" crossorigin="anonymous"/><link rel="apple-touch-icon" sizes="48x48" href="/icons/icon-48x48.png?v=f730a0b8e533f2d4da7bd901d2b7ab80"/><link rel="apple-touch-icon" sizes="72x72" href="/icons/icon-72x72.png?v=f730a0b8e533f2d4da7bd901d2b7ab80"/><link rel="apple-touch-icon" sizes="96x96" href="/icons/icon-96x96.png?v=f730a0b8e533f2d4da7bd901d2b7ab80"/><link rel="apple-touch-icon" sizes="144x144" href="/icons/icon-144x144.png?v=f730a0b8e533f2d4da7bd901d2b7ab80"/><link rel="apple-touch-icon" sizes="192x192" href="/icons/icon-192x192.png?v=f730a0b8e533f2d4da7bd901d2b7ab80"/><link rel="apple-touch-icon" sizes="256x256" href="/icons/icon-256x256.png?v=f730a0b8e533f2d4da7bd901d2b7ab80"/><link rel="apple-touch-icon" sizes="384x384" href="/icons/icon-384x384.png?v=f730a0b8e533f2d4da7bd901d2b7ab80"/><link rel="apple-touch-icon" sizes="512x512" href="/icons/icon-512x512.png?v=f730a0b8e533f2d4da7bd901d2b7ab80"/><link rel="sitemap" type="application/xml" href="/sitemap.xml"/><link rel="canonical" href="https://rocketdocs.netlify.com/usage/arch" data-baseprotocol="https:" data-basehost="rocketdocs.netlify.com"/><link as="script" rel="preload" href="/webpack-runtime-4c66ddc09838456ab29f.js"/><link as="script" rel="preload" href="/framework-305b3707783ccc9d7ca6.js"/><link as="script" rel="preload" href="/f0e45107-02a22ac72de90821f3f0.js"/><link as="script" rel="preload" href="/app-6f502540d49f2ff8e8f7.js"/><link as="script" rel="preload" href="/545f34e4-fd6533567525999d763e.js"/><link as="script" rel="preload" href="/5e2a4920-a9a1a0777b8b3bf4672a.js"/><link as="script" rel="preload" href="/252f366e-3307f783f1a8f590aa23.js"/><link as="script" rel="preload" href="/a7629950cb7a18c7593baf47d0acfe798f1cddc4-2745402a1facfb7fde52.js"/><link as="script" rel="preload" href="/component---node-modules-rocketseat-gatsby-theme-docs-core-src-templates-docs-query-js-11472d50575b2c48f0f9.js"/><link as="fetch" rel="preload" href="/page-data/usage/arch/page-data.json" crossorigin="anonymous"/><link as="fetch" rel="preload" href="/page-data/sq/d/1954253342.json" crossorigin="anonymous"/><link as="fetch" rel="preload" href="/page-data/sq/d/2328931024.json" crossorigin="anonymous"/><link as="fetch" rel="preload" href="/page-data/sq/d/2501019404.json" crossorigin="anonymous"/><link as="fetch" rel="preload" href="/page-data/sq/d/973074209.json" crossorigin="anonymous"/><link as="fetch" rel="preload" href="/page-data/app-data.json" crossorigin="anonymous"/></head><body><div id="___gatsby"><style data-emotion-css="6ifimc">*,*::after,*::before{margin:0;padding:0;box-sizing:border-box;}body{font-size:16px;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,Oxygen,Ubuntu,Cantarell,'Open Sans','Helvetica Neue',sans-serif;background-color:#FFFFFF;text-rendering:optimizelegibility;-webkit-font-smoothing:antialiased;overflow-y:scroll;}h1{font-size:32px;color:#3D3D4D;font-weight:bold;margin-bottom:24px;}h2{font-size:24px;}h3{font-size:18px;}h4{font-size:16px;}h2,h3,h4,h5,h6{color:#3D3D4D;margin:24px 0 16px 0;font-weight:bold;}p{color:#6C6C80;font-size:16px;line-height:28px;margin-bottom:16px;font-weight:400;}code.inline-code{display:inline-block;vertical-align:middle;line-height:1;padding:0.2em;background-color:#44475a;color:rgba(248,248,242);font-size:14px;border-radius:3px;font-feature-settings:'clig' 0,'calt' 0;font-variant:no-common-ligatures no-discretionary-ligatures no-historical-ligatures no-contextual;}h1 code.inline-code,h2 code.inline-code{font-size:calc(100% - 5px);padding:4px;}a{color:#6C6C80;font-weight:bold;-webkit-text-decoration:none;text-decoration:none;}a:hover{-webkit-text-decoration:underline;text-decoration:underline;}blockquote{margin-bottom:16px;width:100%;}blockquote p{padding:1rem;border-radius:5px;background:#F2F2FA;color:#6C6C80;margin:0;}hr{border:0;height:0;border-top:1px solid rgba(0,0,0,0.1);border-bottom:1px solid rgba(255,255,255,0.3);}table{border-collapse:separate;border-spacing:0 4px;margin-top:-4px;margin-bottom:16px;width:100%;}table th,table td{margin:0;color:#6C6C80;background-color:#F2F2FA;border:solid 1px #F2F2FA;border-style:solid none;padding:12px;}table th:first-of-type,table td:first-of-type{border-left-style:solid;border-top-left-radius:5px;border-bottom-left-radius:5px;}table th:last-child,table td:last-child{border-right-style:solid;border-bottom-right-radius:5px;border-top-right-radius:5px;}table tr th{color:#3D3D4D;text-align:left;font-weight:bold;}iframe{margin-bottom:16px;}img{max-width:100%;}ul,ol{color:#6C6C80;padding-left:15px;margin-bottom:16px;}ul li,ol li{line-height:28px;}.gatsby-highlight{font-family:Hack,SFMono-Regular,Menlo,Monaco,Consolas,'Liberation Mono','Courier New',monospace;font-variant:no-common-ligatures no-discretionary-ligatures no-historical-ligatures no-contextual;position:relative;z-index:0;margin:0 0 16px 0;overflow:auto;}.gatsby-highlight .token{font-style:normal !important;}pre[class*='language-'] code{font-family:inherit;}pre[class*='language-']::before{background:#d9d7e0;border-radius:0 0 4px 4px;color:#232129;font-size:12px;font-family:inherit;-webkit-letter-spacing:0.075em;-moz-letter-spacing:0.075em;-ms-letter-spacing:0.075em;letter-spacing:0.075em;line-height:1;padding:0.25rem 0.5rem;position:absolute;left:1rem;text-align:right;text-transform:uppercase;top:0;}pre[class~='language-js']::before,pre[class~='language-javascript']::before{content:'js';background:#f7df1e;}pre[class~='language-jsx']::before{content:'jsx';background:#61dafb;}pre[class~='language-typescript']::before,pre[class~='language-ts']::before{content:'ts';background:#294e80;color:#fff;}pre[class~='language-tsx']::before{content:'tsx';background:#294e80;color:#fff;}pre[class~='language-graphql']::before{content:'GraphQL';background:#e10098;color:#fff;}pre[class~='language-html']::before{content:'html';background:#005a9c;color:#fff;}pre[class~='language-css']::before{content:'css';background:#ff9800;color:#fff;}pre[class~='language-mdx']::before{content:'mdx';background:#f9ac00;color:#fff;}pre[class~='language-shell']::before{content:'shell';}pre[class~='language-sh']::before{content:'sh';}pre[class~='language-bash']::before{content:'bash';}pre[class~='language-yaml']::before,pre[class~='language-yml']::before{content:'yaml';background:#ffa8df;}pre[class~='language-markdown']::before{content:'md';}pre[class~='language-json']::before,pre[class~='language-json5']::before{content:'json';background:linen;}pre[class~='language-diff']::before{content:'diff';background:#e6ffed;}pre[class~='language-text']::before{content:'text';background:#fff;}pre[class~='language-flow']::before{content:'flow';background:#e8bd36;}</style><div style="outline:none" tabindex="-1" id="gatsby-focus-wrapper"><style data-emotion-css="7gspsd">.css-7gspsd::after{content:'';position:fixed;width:100%;height:100%;left:0;right:0;bottom:0;background-color:rgba(0,0,0,0.8);z-index:999;display:none;}</style><div aria-label="Close menu" class="css-7gspsd"></div><style data-emotion-css="1i3ij69">.css-1i3ij69{width:100%;max-width:1400px;margin:0 auto;padding:0 24px;display:grid;grid-template-columns:280px calc(100% - 320px);grid-auto-flow:row;grid-gap:40px;}@media (max-width:780px){.css-1i3ij69{padding:24px;grid-template-columns:100%;}}</style><div class="css-1i3ij69 eees1d50"><style data-emotion-css="efco39">.css-efco39{width:100%;overflow-y:auto;display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-pack:justify;-webkit-justify-content:space-between;-ms-flex-pack:justify;justify-content:space-between;-webkit-align-items:center;-webkit-box-align:center;-ms-flex-align:center;align-items:center;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column;position:-webkit-sticky;position:sticky;top:0;padding-top:36px;-webkit-transition:-webkit-transform 0.5s;-webkit-transition:transform 0.5s;transition:transform 0.5s;height:calc(100vh - 1px);}.css-efco39 nav{width:100%;padding-top:24px;-webkit-align-self:flex-start;-ms-flex-item-align:start;align-self:flex-start;-webkit-flex:1;-ms-flex:1;flex:1;}@media (max-width:780px){.css-efco39{max-width:75%;min-width:auto;z-index:1001;position:fixed;top:0;bottom:0;left:0;padding-top:32px;background:#FFFFFF;-webkit-transform:translate3d( -100%,0,0 );-ms-transform:translate3d( -100%,0,0 );transform:translate3d( -100%,0,0 );}}</style><aside class="css-efco39 e1wh78ri0"><style data-emotion-css="2yrou">.css-2yrou{width:100%;}.css-2yrou a{width:100%;padding-left:30px;display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-pack:start;-webkit-justify-content:flex-start;-ms-flex-pack:start;justify-content:flex-start;-webkit-align-items:center;-webkit-box-align:center;-ms-flex-align:center;align-items:center;}</style><div class="css-2yrou e1wh78ri1"><a aria-label="Go to home page" href="/"><img src="/static/logo-f730a0b8e533f2d4da7bd901d2b7ab80.png" alt="Logo" width="90px" height="80px"/></a></div><nav><style data-emotion-css="mbmci8">.css-mbmci8{list-style:none;width:100%;padding-left:0;display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-pack:start;-webkit-justify-content:flex-start;-ms-flex-pack:start;justify-content:flex-start;-webkit-align-items:center;-webkit-box-align:center;-ms-flex-align:center;align-items:center;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column;}</style><ul class="css-mbmci8 e1wh78ri2"><style data-emotion-css="1wldk6c">.css-1wldk6c{font-size:15px;width:100%;-webkit-transition:all 200ms ease-in-out;transition:all 200ms ease-in-out;padding:0 20px;cursor:pointer;}.css-1wldk6c a,.css-1wldk6c span{display:block;font-size:15px;color:#6C6C80;background-color:#FFFFFF;padding:4px 10px;margin:4px 0;border-radius:4px;font-weight:normal;-webkit-text-decoration:none;text-decoration:none;width:100%;height:100%;display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-pack:start;-webkit-justify-content:flex-start;-ms-flex-pack:start;justify-content:flex-start;-webkit-align-items:center;-webkit-box-align:center;-ms-flex-align:center;align-items:center;cursor:pointer;margin:0 auto;-webkit-transition:all 0.2s ease;transition:all 0.2s ease;}.css-1wldk6c a svg,.css-1wldk6c span svg{width:20px;height:20px;margin-right:10px;}.css-1wldk6c a:not(.active-link):hover,.css-1wldk6c span:not(.active-link):hover{color:#8257E6;}.css-1wldk6c a.active-link,.css-1wldk6c span.active-link{color:#8257E6;background-color:#F2F2FA;}@media (max-width:780px){.css-1wldk6c a.active-link,.css-1wldk6c span.active-link{background:#F2F2FA;}}</style><li class="css-1wldk6c e1wh78ri4"><a href="/">Introduction</a></li><li class="css-1wldk6c e1wh78ri4"><a href="https://github.com/apache/incubator-liminal/blob/master/README.md" rel="noopener noreferrer">Getting started<svg stroke="currentColor" fill="none" stroke-width="2" viewBox="0 0 24 24" stroke-linecap="round" stroke-linejoin="round" style="width:16px;height:16px;margin-left:10px" height="1em" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M18 13v6a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V8a2 2 0 0 1 2-2h6"></path><polyline points="15 3 21 3 21 9"></polyline><line x1="10" y1="14" x2="21" y2="3"></line></svg></a></li><li class="css-1wldk6c e1wh78ri4"><a aria-current="page" class="active-link" href="/usage/arch">Architecture</a></li><style data-emotion-css="mximbs">.css-mximbs{padding-left:30px;width:100%;text-transform:uppercase;font-size:14px;font-weight:bold;margin-top:20px;color:#3D3D4D;-webkit-letter-spacing:0.142em;-moz-letter-spacing:0.142em;-ms-letter-spacing:0.142em;letter-spacing:0.142em;}</style><li class="css-mximbs e1wh78ri3">Community</li><style data-emotion-css="157nhwy">.css-157nhwy{list-style:none;width:100%;padding-left:0;display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-pack:start;-webkit-justify-content:flex-start;-ms-flex-pack:start;justify-content:flex-start;-webkit-align-items:center;-webkit-box-align:center;-ms-flex-align:center;align-items:center;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column;margin:5px 0 0 0;}</style><ul class="css-157nhwy e1wh78ri5"><li class="css-1wldk6c e1wh78ri4"><a href="https://github.com/apache/incubator-liminal" rel="noopener noreferrer">Github<svg stroke="currentColor" fill="none" stroke-width="2" viewBox="0 0 24 24" stroke-linecap="round" stroke-linejoin="round" style="width:16px;height:16px;margin-left:10px" height="1em" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M18 13v6a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V8a2 2 0 0 1 2-2h6"></path><polyline points="15 3 21 3 21 9"></polyline><line x1="10" y1="14" x2="21" y2="3"></line></svg></a></li><li class="css-1wldk6c e1wh78ri4"><a href="https://lists.apache.org/list.html?dev@liminal.apache.org" rel="noopener noreferrer">Dev-Mailing-List<svg stroke="currentColor" fill="none" stroke-width="2" viewBox="0 0 24 24" stroke-linecap="round" stroke-linejoin="round" style="width:16px;height:16px;margin-left:10px" height="1em" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M18 13v6a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V8a2 2 0 0 1 2-2h6"></path><polyline points="15 3 21 3 21 9"></polyline><line x1="10" y1="14" x2="21" y2="3"></line></svg></a></li></ul></ul></nav></aside><style data-emotion-css="19vacph">.css-19vacph{height:100%;padding-top:36px;width:100%;display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-pack:start;-webkit-justify-content:flex-start;-ms-flex-pack:start;justify-content:flex-start;position:relative;}@media (max-width:1200px){.css-19vacph{-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column;}}@media (max-width:780px){.css-19vacph{padding-top:0;}}</style><main class="css-19vacph eees1d51"><style data-emotion-css="9q3p0l">.css-9q3p0l{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-pack:start;-webkit-justify-content:flex-start;-ms-flex-pack:start;justify-content:flex-start;-webkit-align-items:center;-webkit-box-align:center;-ms-flex-align:center;align-items:center;height:40px;margin-bottom:24px;}.css-9q3p0l h2{margin:0;border:none;padding:0;font-size:18px;}@media (max-width:359px){.css-9q3p0l h2{font-size:14px;}}.css-9q3p0l button{border:none;background:none;cursor:pointer;margin-right:16px;display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-pack:start;-webkit-justify-content:flex-start;-ms-flex-pack:start;justify-content:flex-start;-webkit-align-items:center;-webkit-box-align:center;-ms-flex-align:center;align-items:center;}@media (min-width:780px){.css-9q3p0l{display:none;}}</style><header class="css-9q3p0l esnpb7p0"><button aria-label="Open sidebar" type="button"><svg stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 512 512" size="23" aria-hidden="true" height="23" width="23" xmlns="http://www.w3.org/2000/svg"><path d="M32 96v64h448V96H32zm0 128v64h448v-64H32zm0 128v64h448v-64H32z"></path></svg></button><h2></h2></header><style data-emotion-css="1ehkd4h">.css-1ehkd4h{width:100%;max-width:calc(75% - 64px);padding-right:64px;}@media (max-width:1200px){.css-1ehkd4h{max-width:100%;padding-right:0;-webkit-order:3;-ms-flex-order:3;order:3;}}</style><div class="css-1ehkd4h eees1d52"><h2 id="motivation" style="position:relative"><a href="#motivation" aria-label="motivation permalink" class="anchor before"><svg aria-hidden="true" focusable="false" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Motivation</h2><p>The challenges involved in operationalizing machine learning models are one of the main reasons why many machine learning projects never make it to production. The process involves automating and orchestrating multiple steps which run on heterogeneous infrastructure - different compute environments, data processing platforms, ML frameworks, notebooks, containers and monitoring tools. There are no mature standards for this workflow, and most organizations do not have the experience to build it in-house. In the best case, dev-ds-devops teams form in order to accomplish this task together; in many cases, it&#x27;s the data scientists who try to deal with this themselves without the knowledge or the inclination to become infrastructure experts. As a result, many projects never make it through the cycle. Those who do suffer from a very long lead time from a successful experiment to an operational, refreshable, deployed and monitored model in production.</p><p>Apache Liminal was created to enable scalability in ML efforts and after a thorough review of available solutions and frameworks, which did not meet our main KPIs. Few Commercial &amp; Open-source solutions have started to emerge in the last few years, however, they are limited to specific parts of the workflow, such as Databricks MLFlow or tied to a specific environment (e.g. SageMaker on AWS) or a specific tech stack (e.g. KubeFlow).</p><h2 id="liminal-workflows" style="position:relative"><a href="#liminal-workflows" aria-label="liminal workflows permalink" class="anchor before"><svg aria-hidden="true" focusable="false" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Liminal Workflows</h2><p>Apache Liminal provides declarative building blocks which define the workflow, orchestrate the underlying infrastructure and takes care of non functional concerns; thus enabling focus in business logic / algorithm code. Liminal approach is to wrap existing frameworks and libraries with a DSL (Domain-Specific-Language), which is used by data engineers &amp; scientists to express ML/AI workflows using the right abstractions; from model training to real time inference in production.</p><p>The architecture enables and promotes adoption of specific components in existing (non-Liminal) frameworks, as well as seamless integration with other open source projects. The following diagram depicts these main components and where Apache Liminal comes in:</p><img alt="Liminal" src="/liminal_01.png"/><p>A classical data scientist workflow includes some base phases: Train, Deploy and Consume.</p><p><em>The Train phase includes the following tasks:</em></p><ul><li>Fetch - get the data needed to build a model - usually using SQL</li><li>Clean - make sure the data is useful for building the model</li><li>Prepare - extract features from the data</li><li>Train - build the model and tune it</li><li>Evaluate - test model integrity</li><li>Validate - test model quality</li></ul><p><em>The Deploy phase includes these tasks:</em></p><ul><li>Deploy - make it available for usage in production</li><li>Inference - The actual use of the models created by applications and ETLs, usually through APIs to the batch or real-time inference that usually rely on Model and Feature stores.</li><li>Monitor - consistent monitoring of ML models performance and KPIs</li></ul><p>Liminal provides its users a declarative composition capabilities to materialize these steps in a robust way, using standard ML/AI frameworks, tools &amp; libraries, e.g. scikit-learn, Pandas, Airfolow, Kubernetes, MLFLow etc. Liminal&#x27;s plugin architecture provides an easy way to extend and evolve the platform.</p><img alt="Liminal" src="/liminal_02.png"/><style data-emotion-css="1rfih1g">.css-1rfih1g{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-pack:justify;-webkit-justify-content:space-between;-ms-flex-pack:justify;justify-content:space-between;-webkit-align-items:center;-webkit-box-align:center;-ms-flex-align:center;align-items:center;padding:48px 0;width:100%;}@media (max-width:780px){.css-1rfih1g{-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column;}}</style><section class="css-1rfih1g e1jikabl0"><style data-emotion-css="asqskg">.css-asqskg{-webkit-transition:all 200ms;transition:all 200ms;}.css-asqskg a{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-text-decoration:none;text-decoration:none;-webkit-box-pack:center;-webkit-justify-content:center;-ms-flex-pack:center;justify-content:center;-webkit-align-items:center;-webkit-box-align:center;-ms-flex-align:center;align-items:center;width:100%;height:100%;}.css-asqskg a svg{width:25px;height:25px;color:#6C6C80;margin-right:16px;}.css-asqskg a p{-webkit-letter-spacing:0.142em;-moz-letter-spacing:0.142em;-ms-letter-spacing:0.142em;letter-spacing:0.142em;text-transform:uppercase;font-size:12px;margin:0;color:#6C6C80;}.css-asqskg a h3{color:#6C6C80;border:none;margin:0;padding:0;font-size:16px;}.css-asqskg:hover{opacity:0.8;}.css-asqskg:hover a svg{opacity:0.8;}@media (max-width:780px){.css-asqskg{width:100%;margin-bottom:16px;}.css-asqskg a{-webkit-box-pack:start;-webkit-justify-content:flex-start;-ms-flex-pack:start;justify-content:flex-start;}}</style><div class="css-asqskg e1jikabl1"><a href="https://github.com/apache/incubator-liminal/blob/master/README.md"><svg stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 24 24" height="1em" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M15.41 16.09l-4.58-4.59 4.58-4.59L14 5.5l-6 6 6 6z"></path></svg><div><p>Prev</p><h3>Getting started</h3></div></a></div><style data-emotion-css="3wkrrt">.css-3wkrrt{-webkit-transition:all 200ms;transition:all 200ms;margin-left:auto;}.css-3wkrrt a{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-text-decoration:none;text-decoration:none;-webkit-box-pack:center;-webkit-justify-content:center;-ms-flex-pack:center;justify-content:center;-webkit-align-items:center;-webkit-box-align:center;-ms-flex-align:center;align-items:center;width:100%;height:100%;}.css-3wkrrt a svg{width:25px;height:25px;color:#6C6C80;margin-left:16px;}.css-3wkrrt a p{-webkit-letter-spacing:0.142em;-moz-letter-spacing:0.142em;-ms-letter-spacing:0.142em;letter-spacing:0.142em;text-transform:uppercase;font-size:12px;margin:0;color:#6C6C80;}.css-3wkrrt a h3{color:#6C6C80;border:none;margin:0;padding:0;font-size:16px;}.css-3wkrrt:hover{opacity:0.8;}.css-3wkrrt:hover a svg{opacity:0.8;}@media (max-width:780px){.css-3wkrrt{width:100%;}.css-3wkrrt a{-webkit-box-pack:end;-webkit-justify-content:flex-end;-ms-flex-pack:end;justify-content:flex-end;}}</style><div class="css-3wkrrt e1jikabl1"><a href="https://github.com/apache/incubator-liminal"><div><p>Next</p><h3>Github</h3></div><svg stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 24 24" height="1em" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M8.59 16.34l4.58-4.59-4.58-4.59L10 5.75l6 6-6 6z"></path></svg></a></div></section></div><style data-emotion-css="15t48j3">.css-15t48j3{position:-webkit-sticky;position:sticky;top:0;-webkit-order:2;-ms-flex-order:2;order:2;padding-top:72px;max-height:calc(100vh - 72px);overflow-y:auto;width:100%;max-width:200px;}@media (max-width:1200px){.css-15t48j3{position:relative;top:auto;-webkit-order:0;-ms-flex-order:0;order:0;max-width:100%;margin-left:0;padding-top:0;}}</style><div class="css-15t48j3 e10o0eax0"><style data-emotion-css="157u13d">.css-157u13d h2{text-transform:uppercase;font-size:14px;font-weight:bold;-webkit-letter-spacing:0.142em;-moz-letter-spacing:0.142em;-ms-letter-spacing:0.142em;letter-spacing:0.142em;margin-top:0rem;border:none;margin:0 0 16px 0;}.css-157u13d nav ul{padding-left:0;margin-bottom:0;list-style:none;}.css-157u13d nav ul li{margin-bottom:12px;line-height:1.1;}.css-157u13d nav ul li a{font-size:14px;font-weight:400;-webkit-text-decoration:none;text-decoration:none;-webkit-transition:all 0.2s;transition:all 0.2s;}.css-157u13d nav ul li a:hover{-webkit-text-decoration:underline;text-decoration:underline;}@media (max-width:1200px){.css-157u13d{margin:0 0 24px 0;padding-bottom:16px;border-bottom:1px solid rgba(120,117,122,0.2);}}</style><div class="css-157u13d e10o0eax1"><h2>On this page</h2><nav><ul><li><a href="#motivation" style="color:#6C6C80">Motivation</a></li><li><a href="#liminal-workflows" style="color:#6C6C80">Liminal Workflows</a></li></ul></nav></div></div></main></div></div><div id="gatsby-announcer" style="position:absolute;top:0;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border:0" aria-live="assertive" aria-atomic="true"></div></div><script async="" src="https://www.googletagmanager.com/gtag/js?id=G-SSE7V5RL26"></script><script>
    window.GATSBY_GTAG_PLUGIN_GA_TRACKING_ID = (
      'G-SSE7V5RL26'
    );
    window.GATSBY_GTAG_PLUGIN_ANONYMIZE = false;

    var options = {
      send_page_view: false
    };
    if (false) {
      options.anonymize_ip = true;
    }

    window.dataLayer = window.dataLayer || [];
    function gtag(){dataLayer.push(arguments);}
    window.gtag = gtag;
    gtag('js', new Date());
    gtag('config', 'G-SSE7V5RL26', options);
  </script><script id="gatsby-script-loader">/*<![CDATA[*/window.pagePath="/usage/arch";/*]]>*/</script><script id="gatsby-chunk-mapping">/*<![CDATA[*/window.___chunkMapping={"polyfill":["/polyfill-5204fa10d5950c3006a4.js"],"app":["/app-6f502540d49f2ff8e8f7.js"],"component---cache-caches-gatsby-plugin-offline-app-shell-js":["/component---cache-caches-gatsby-plugin-offline-app-shell-js-241020a1cf7d0f317bbe.js"],"component---node-modules-rocketseat-gatsby-theme-docs-core-src-templates-docs-query-js":["/component---node-modules-rocketseat-gatsby-theme-docs-core-src-templates-docs-query-js-11472d50575b2c48f0f9.js"],"component---node-modules-rocketseat-gatsby-theme-docs-core-src-templates-homepage-query-js":["/component---node-modules-rocketseat-gatsby-theme-docs-core-src-templates-homepage-query-js-a7d6ef8d07b6353dab59.js"],"component---src-pages-404-js":["/component---src-pages-404-js-587ddd94f1ae754b631c.js"]};/*]]>*/</script><script src="/polyfill-5204fa10d5950c3006a4.js" nomodule=""></script><script src="/component---node-modules-rocketseat-gatsby-theme-docs-core-src-templates-docs-query-js-11472d50575b2c48f0f9.js" async=""></script><script src="/a7629950cb7a18c7593baf47d0acfe798f1cddc4-2745402a1facfb7fde52.js" async=""></script><script src="/252f366e-3307f783f1a8f590aa23.js" async=""></script><script src="/5e2a4920-a9a1a0777b8b3bf4672a.js" async=""></script><script src="/545f34e4-fd6533567525999d763e.js" async=""></script><script src="/app-6f502540d49f2ff8e8f7.js" async=""></script><script src="/f0e45107-02a22ac72de90821f3f0.js" async=""></script><script src="/framework-305b3707783ccc9d7ca6.js" async=""></script><script src="/webpack-runtime-4c66ddc09838456ab29f.js" async=""></script></body></html>