| <!DOCTYPE html> |
| <html lang="en-US"> |
| |
| <head> |
| <meta charset="utf-8"> |
| <meta http-equiv="X-UA-Compatible" content="IE=edge"> |
| <meta name="viewport" content="width=device-width, initial-scale=1"> |
| |
| <link rel="stylesheet" href="/assets/css/main.css"> |
| <link rel="stylesheet" href="/assets/css/font-awesome.min.css"> |
| |
| <link rel="shortcut icon" href="/favicon.ico?1"> |
| <!-- Begin Jekyll SEO tag v2.8.0 --> |
| <title>Runtime Design | Nemo</title> |
| <meta name="generator" content="Jekyll v3.9.2" /> |
| <meta property="og:title" content="Runtime Design" /> |
| <meta property="og:locale" content="en_US" /> |
| <meta name="description" content="Receiving a Job from the Nemo Compiler" /> |
| <meta property="og:description" content="Receiving a Job from the Nemo Compiler" /> |
| <link rel="canonical" href="http://nemo.apache.org//docs/runtime_design/" /> |
| <meta property="og:url" content="http://nemo.apache.org//docs/runtime_design/" /> |
| <meta property="og:site_name" content="Nemo" /> |
| <meta property="og:type" content="article" /> |
| <meta property="article:published_time" content="2022-09-09T21:18:53+09:00" /> |
| <meta name="twitter:card" content="summary" /> |
| <meta property="twitter:title" content="Runtime Design" /> |
| <script type="application/ld+json"> |
| {"@context":"https://schema.org","@type":"WebPage","dateModified":"2022-09-09T21:18:53+09:00","datePublished":"2022-09-09T21:18:53+09:00","description":"Receiving a Job from the Nemo Compiler","headline":"Runtime Design","url":"http://nemo.apache.org//docs/runtime_design/"}</script> |
| <!-- End Jekyll SEO tag --> |
| |
| |
| <link rel="canonical" href="http://nemo.apache.org//docs/runtime_design/"> |
| <link rel="alternate" type="application/rss+xml" title="Nemo" href="http://nemo.apache.org//feed.xml" /> |
| </head> |
| |
| |
| <body> |
| |
| <nav class="navbar navbar-default navbar-fixed-top"> |
| <div class="container navbar-container"> |
| <div class="navbar-header"> |
| <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar"> |
| <span class="sr-only">Toggle navigation</span> |
| <span class="icon-bar"></span> |
| <span class="icon-bar"></span> |
| <span class="icon-bar"></span> |
| </button> |
| <a class="navbar-brand" href="/"> |
| <span><img src="/assets/img/nemo-logo.png" alt="Logo"></span> |
| </a> |
| </div> |
| <div id="navbar" class="collapse navbar-collapse"> |
| <ul class="nav navbar-nav"> |
| <li class="active" ><a href="/docs/home/">Docs</a></li> |
| <li ><a href="/apidocs">APIs</a></li> |
| <li ><a href="/pages/downloads">Downloads</a></li> |
| <li ><a href="/pages/talks">Talks</a></li> |
| <li ><a href="/pages/team">Team</a></li> |
| <li ><a href="/pages/license">License</a></li> |
| <li ><a href="/blog/2020/03/09/release-note-0.2/">Blog</a></li> |
| </ul> |
| <div class="navbar-right"> |
| <form class="navbar-form navbar-left"> |
| <div class="form-group has-feedback"> |
| <input id="search-box" type="search" class="form-control" placeholder="Search..."> |
| <i class="fa fa-search form-control-feedback"></i> |
| </div> |
| </form> |
| <ul class="nav navbar-nav"> |
| <li><a href="https://github.com/apache/incubator-nemo"><i class="fa fa-github" aria-hidden="true"></i></a></li> |
| </ul> |
| </div> |
| </div> |
| </div> |
| </nav> |
| |
| |
| <div class="page-content"> |
| <div class="wrapper"> |
| |
| <div class="container"> |
| <div class="row"> |
| <div class="col-md-12"> |
| <div class="container"> |
| <div class="row"> |
| <div class="col-md-4"> |
| <div class="panel-group" id="accordion" role="tablist" aria-multiselectable="true"> |
| |
| <div class="panel panel-default"> |
| <div class="panel-heading"> |
| <h4 class="panel-title"> |
| <a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapse-1" aria-expanded="false" aria-controls="collapse-1"> |
| Getting Started |
| </a> |
| </h4> |
| </div> |
| |
| |
| <div class="panel-collapse collapse" id="collapse-1" role="tabpanel" aria-label="Side Navigation"> |
| <div class="list-group"> |
| |
| |
| |
| <a class="list-group-item " href="/docs/home/">Overview</a> |
| |
| |
| |
| <a class="list-group-item " href="/docs/getting_started/">Getting Started</a> |
| |
| </div> |
| </div> |
| </div> |
| |
| <div class="panel panel-default"> |
| <div class="panel-heading"> |
| <h4 class="panel-title"> |
| <a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapse-2" aria-expanded="false" aria-controls="collapse-2"> |
| Optimizations |
| </a> |
| </h4> |
| </div> |
| |
| |
| <div class="panel-collapse collapse" id="collapse-2" role="tabpanel" aria-label="Side Navigation"> |
| <div class="list-group"> |
| |
| |
| |
| <a class="list-group-item " href="/docs/ir/">Nemo Intermediate Representation (IR)</a> |
| |
| |
| |
| <a class="list-group-item " href="/docs/passes_and_policies/">Passes and Policies</a> |
| |
| </div> |
| </div> |
| </div> |
| |
| <div class="panel panel-default"> |
| <div class="panel-heading"> |
| <h4 class="panel-title"> |
| <a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapse-3" aria-expanded="false" aria-controls="collapse-3"> |
| System Designs |
| </a> |
| </h4> |
| </div> |
| |
| |
| <div class="panel-collapse" id="collapse-3" role="tabpanel" aria-label="Side Navigation"> |
| <div class="list-group"> |
| |
| |
| |
| <a class="list-group-item " href="/docs/compiler_design/">Compiler Design</a> |
| |
| |
| |
| <a class="list-group-item active" href="/docs/runtime_design/">Runtime Design</a> |
| |
| </div> |
| </div> |
| </div> |
| |
| <div class="panel panel-default"> |
| <div class="panel-heading"> |
| <h4 class="panel-title"> |
| <a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapse-4" aria-expanded="false" aria-controls="collapse-4"> |
| Contribute |
| </a> |
| </h4> |
| </div> |
| |
| |
| <div class="panel-collapse collapse" id="collapse-4" role="tabpanel" aria-label="Side Navigation"> |
| <div class="list-group"> |
| |
| |
| |
| <a class="list-group-item " href="/docs/contribute/">Contribute</a> |
| |
| </div> |
| </div> |
| </div> |
| |
| <div class="panel panel-default"> |
| <div class="panel-heading"> |
| <h4 class="panel-title"> |
| <a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapse-5" aria-expanded="false" aria-controls="collapse-5"> |
| Security |
| </a> |
| </h4> |
| </div> |
| |
| |
| <div class="panel-collapse collapse" id="collapse-5" role="tabpanel" aria-label="Side Navigation"> |
| <div class="list-group"> |
| |
| |
| |
| <a class="list-group-item " href="/docs/security/">Security Guide</a> |
| |
| </div> |
| </div> |
| </div> |
| |
| </div> |
| |
| </div> |
| |
| <div class="col-md-8"> |
| <h1>Runtime Design</h1> |
| <div id="markdown-content-container"><h3 id="receiving-a-job-from-the-nemo-compiler">Receiving a Job from the Nemo Compiler</h3> |
| |
| <p>After the compiler goes through a set of passes for optimization, the optimized Nemo IR is translated into into a |
| physical form for the execution runtime to execute. This involves translations like expanding an operator annotated |
| with parallelism in Nemo IR to the desired number of tasks and connecting the tasks according to the data communication |
| patterns annotated on the IR edges. Physical execution plan is also in the form of a DAG, with the same values annotated |
| for execution properties as the given IR DAG if necessary. Nemo IR DAG and physical execution plan can be translated |
| from one another by sharing the identifiers.</p> |
| |
| <h3 id="runtime-architecture">Runtime Architecture</h3> |
| <p>The Nemo runtime consists of a <em>RuntimeMaster</em> and multiple <em>Executors</em>. |
| <em>RuntimeMaster</em> takes the submitted physical execution plan and schedules each <em>TaskGroup</em> to <em>Executor</em> for execution.</p> |
| |
| <p>The figure below shows the Nemo runtime’s overall architecture. |
| Our runtime’s components can be broken down into two parts, the processing backbone and the extensible modules.</p> |
| |
| <p>The processing backbone illustrated by the blue double stroked boxes in the figure below, |
| implements the inherent and basic code that must be executed for all Nemo jobs |
| (and potentially all data processing jobs). |
| The code includes references to the flexible and extensible data structures |
| representing our execution properties. |
| For example, “executor placement” is mainly controlled in ContainerManager as an extensible map.</p> |
| |
| <p>The extensible modules illustrated by the red dashed boxes in the figure below, |
| are interfaces which users can implement in order to get them to behave the way the user wants. |
| Each interface has been transparently integrated with other components of the runtime.</p> |
| |
| <p><img src="/assets/runtime_arch.png" alt="image" /></p> |
| |
| <h3 id="dictionary">Dictionary</h3> |
| <ul> |
| <li>Stage: A unit of execution the runtime uses for scheduling the job.</li> |
| <li>TaskGroup: A computation unit composed of one or more tasks that can be computed in a single executor.</li> |
| <li>Block: The unit of data output by a single task.</li> |
| <li>Partition: A block consists of one or more partitions, depending on the <em>Partitioner</em> choice.</li> |
| </ul> |
| |
| </div> |
| <div style="clear:both;"> |
| <p class="text-center"> |
| <br /> |
| <a target="_blank" href="https://github.com/apache/incubator-nemo-website/_docs/designs/runtime_design.md" class="btn btn-default" role="button"> |
| <i class="fa fa-pencil fa-lg"></i> Improve this page |
| </a> |
| </p> |
| </div> |
| <hr> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <ul class="pager"> |
| |
| |
| |
| <li class="previous"> |
| <a href="/docs/compiler_design/"> |
| <span aria-hidden="true">←</span> Previous |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| <li class="next"> |
| <a href="/docs/contribute/"> |
| Next <span aria-hidden="true">→</span> |
| </a> |
| </li> |
| |
| </ul> |
| <div class="clear"></div> |
| |
| |
| </div> |
| |
| </div> |
| </div> |
| |
| </div> |
| </div> |
| </div> |
| </div> |
| </div> |
| |
| <footer class="footer"> |
| <div class="container"> |
| |
| <p class="text-center"> |
| Nemo 2022 | |
| Powered by <a href="https://github.com/aksakalli/jekyll-doc-theme">Jekyll Doc Theme</a> |
| </p> |
| <!-- <p class="text-muted">Place sticky footer content here.</p> --> |
| </div> |
| </footer> |
| |
| <script> |
| var baseurl = '' |
| </script> |
| <script src="https://code.jquery.com/jquery-1.12.4.min.js"></script> |
| <script src="/assets/js/bootstrap.min.js "></script> |
| <script src="/assets/js/typeahead.bundle.min.js "></script> |
| |
| <script src="/assets/js/main.js "></script> |
| |
| </body> |
| |
| </html> |