blob: 48167c44726987d33b56285cd7eb52079a7a278e [file] [log] [blame]
<!DOCTYPE html>
<html lang="en-US">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="/assets/css/custom.css">
<link rel="stylesheet" href="/assets/css/main.css">
<link rel="stylesheet" href="/assets/css/font-awesome.min.css">
<link rel="shortcut icon" href="/favicon.ico?1">
<!-- Begin Jekyll SEO tag v2.8.0 -->
<title>Runtime Design | Nemo</title>
<meta name="generator" content="Jekyll v3.9.3" />
<meta property="og:title" content="Runtime Design" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="Receiving a Job from the Nemo Compiler" />
<meta property="og:description" content="Receiving a Job from the Nemo Compiler" />
<link rel="canonical" href="http://nemo.apache.org//docs/runtime_design/" />
<meta property="og:url" content="http://nemo.apache.org//docs/runtime_design/" />
<meta property="og:site_name" content="Nemo" />
<meta property="og:type" content="article" />
<meta property="article:published_time" content="2023-05-11T14:14:28+09:00" />
<meta name="twitter:card" content="summary" />
<meta property="twitter:title" content="Runtime Design" />
<script type="application/ld+json">
{"@context":"https://schema.org","@type":"WebPage","dateModified":"2023-05-11T14:14:28+09:00","datePublished":"2023-05-11T14:14:28+09:00","description":"Receiving a Job from the Nemo Compiler","headline":"Runtime Design","url":"http://nemo.apache.org//docs/runtime_design/"}</script>
<!-- End Jekyll SEO tag -->
<link rel="canonical" href="http://nemo.apache.org//docs/runtime_design/">
<link rel="alternate" type="application/rss+xml" title="Nemo" href="http://nemo.apache.org//feed.xml" />
</head>
<body>
<nav class="navbar navbar-default navbar-fixed-top">
<div class="container navbar-container">
<div class="navbar-header">
<button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
<span class="sr-only">Toggle navigation</span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
<a class="navbar-brand" href="/">
<span><img src="/assets/img/nemo-logo.png" alt="Logo"></span>
</a>
</div>
<div id="navbar" class="collapse navbar-collapse">
<ul class="nav navbar-nav">
<li class="active" ><a href="/docs/home/">Docs</a></li>
<li ><a href="/apidocs">APIs</a></li>
<li ><a href="/pages/downloads">Downloads</a></li>
<li ><a href="/pages/talks">Talks</a></li>
<li ><a href="/pages/team">Team</a></li>
<li ><a href="/pages/license">License</a></li>
<li ><a href="/blog/2022/09/10/release-note-0.4/">Blog</a></li>
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">ASF<span class="caret"></span></a>
<ul class="dropdown-menu">
<li><a href="http://www.apache.org/">Apache Software Foundation</a></li>
<li><a href="https://www.apache.org/foundation/how-it-works.html">How Apache Works</a></li>
<li><a href="https://www.apache.org/licenses/">License</a></li>
<li><a href="https://www.apache.org/security/">Security</a></li>
<li><a href="https://privacy.apache.org/policies/privacy-policy-public.html">Privacy</a></li>
<li><a href="https://www.apache.org/foundation/sponsorship.html">Sponsoring Apache</a></li>
<li><a href="https://www.apache.org/foundation/thanks.html">Thanks</a></li>
</ul>
</li>
</ul>
<div class="navbar-right">
<form class="navbar-form navbar-left">
<div class="form-group has-feedback">
<input id="search-box" type="search" class="form-control" placeholder="Search...">
<i class="fa fa-search form-control-feedback"></i>
</div>
</form>
<ul class="nav navbar-nav">
<li><a href="https://github.com/apache/incubator-nemo"><i class="fa fa-github" aria-hidden="true"></i></a></li>
</ul>
</div>
</div>
</div>
</nav>
<div class="page-content">
<div class="wrapper">
<div class="container">
<div class="row">
<div class="col-md-4">
<div class="panel-group" id="accordion" role="tablist" aria-multiselectable="true">
<div class="panel panel-default">
<div class="panel-heading">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapse-1" aria-expanded="false" aria-controls="collapse-1">
Getting Started
</a>
</h4>
</div>
<div class="panel-collapse collapse" id="collapse-1" role="tabpanel" aria-label="Side Navigation">
<div class="list-group">
<a class="list-group-item " href="/docs/home/">Overview</a>
<a class="list-group-item " href="/docs/getting_started/">Getting Started</a>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapse-2" aria-expanded="false" aria-controls="collapse-2">
Optimizations
</a>
</h4>
</div>
<div class="panel-collapse collapse" id="collapse-2" role="tabpanel" aria-label="Side Navigation">
<div class="list-group">
<a class="list-group-item " href="/docs/ir/">Nemo Intermediate Representation (IR)</a>
<a class="list-group-item " href="/docs/passes_and_policies/">Passes and Policies</a>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapse-3" aria-expanded="false" aria-controls="collapse-3">
System Designs
</a>
</h4>
</div>
<div class="panel-collapse" id="collapse-3" role="tabpanel" aria-label="Side Navigation">
<div class="list-group">
<a class="list-group-item " href="/docs/compiler_design/">Compiler Design</a>
<a class="list-group-item active" href="/docs/runtime_design/">Runtime Design</a>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapse-4" aria-expanded="false" aria-controls="collapse-4">
Contribute
</a>
</h4>
</div>
<div class="panel-collapse collapse" id="collapse-4" role="tabpanel" aria-label="Side Navigation">
<div class="list-group">
<a class="list-group-item " href="/docs/contribute/">Contribute</a>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapse-5" aria-expanded="false" aria-controls="collapse-5">
Security
</a>
</h4>
</div>
<div class="panel-collapse collapse" id="collapse-5" role="tabpanel" aria-label="Side Navigation">
<div class="list-group">
<a class="list-group-item " href="/docs/security/">Security Guide</a>
</div>
</div>
</div>
</div>
</div>
<div class="col-md-8">
<h1>Runtime Design</h1>
<div id="markdown-content-container"><h3 id="receiving-a-job-from-the-nemo-compiler">Receiving a Job from the Nemo Compiler</h3>
<p>After the compiler goes through a set of passes for optimization, the optimized Nemo IR is translated into into a
physical form for the execution runtime to execute. This involves translations like expanding an operator annotated
with parallelism in Nemo IR to the desired number of tasks and connecting the tasks according to the data communication
patterns annotated on the IR edges. Physical execution plan is also in the form of a DAG, with the same values annotated
for execution properties as the given IR DAG if necessary. Nemo IR DAG and physical execution plan can be translated
from one another by sharing the identifiers.</p>
<h3 id="runtime-architecture">Runtime Architecture</h3>
<p>The Nemo runtime consists of a <em>RuntimeMaster</em> and multiple <em>Executors</em>.
<em>RuntimeMaster</em> takes the submitted physical execution plan and schedules each <em>TaskGroup</em> to <em>Executor</em> for execution.</p>
<p>The figure below shows the Nemo runtime’s overall architecture.
Our runtime’s components can be broken down into two parts, the processing backbone and the extensible modules.</p>
<p>The processing backbone illustrated by the blue double stroked boxes in the figure below,
implements the inherent and basic code that must be executed for all Nemo jobs
(and potentially all data processing jobs).
The code includes references to the flexible and extensible data structures
representing our execution properties.
For example, “executor placement” is mainly controlled in ContainerManager as an extensible map.</p>
<p>The extensible modules illustrated by the red dashed boxes in the figure below,
are interfaces which users can implement in order to get them to behave the way the user wants.
Each interface has been transparently integrated with other components of the runtime.</p>
<p><img src="/assets/img/runtime.jpg" alt="image" /></p>
<h3 id="dictionary">Dictionary</h3>
<ul>
<li>Stage: A unit of execution the runtime uses for scheduling the job.</li>
<li>TaskGroup: A computation unit composed of one or more tasks that can be computed in a single executor.</li>
<li>Block: The unit of data output by a single task.</li>
<li>Partition: A block consists of one or more partitions, depending on the <em>Partitioner</em> choice.</li>
</ul>
</div>
<div style="clear:both;">
<p class="text-center">
<br />
<a target="_blank" href="https://github.com/apache/incubator-nemo-website/blob/asf-site/_docs/designs/runtime_design.md" class="btn btn-default" role="button">
<i class="fa fa-pencil fa-lg"></i> Improve this page
</a>
</p>
</div>
<hr>
<ul class="pager">
<li class="previous">
<a href="/docs/compiler_design/">
<span aria-hidden="true">&larr;</span> Previous
</a>
</li>
<li class="next">
<a href="/docs/contribute/">
Next <span aria-hidden="true">&rarr;</span>
</a>
</li>
</ul>
<div class="clear"></div>
<div id="disqus_thread"></div>
<script>
/**
* RECOMMENDED CONFIGURATION VARIABLES: EDIT AND UNCOMMENT THE SECTION BELOW TO INSERT DYNAMIC VALUES FROM YOUR PLATFORM OR CMS.
* LEARN WHY DEFINING THESE VARIABLES IS IMPORTANT: https://disqus.com/admin/universalcode/#configuration-variables */
/*
var disqus_config = function () {
this.page.url = PAGE_URL; // Replace PAGE_URL with your page's canonical URL variable
this.page.identifier = PAGE_IDENTIFIER; // Replace PAGE_IDENTIFIER with your page's unique identifier variable
};
*/
(function() { // DON'T EDIT BELOW THIS LINE
var d = document, s = d.createElement('script');
s.src = 'https://apache-nemo.disqus.com/embed.js';
s.setAttribute('data-timestamp', +new Date());
(d.head || d.body).appendChild(s);
})();
</script>
<noscript>Please enable JavaScript to view the <a href="https://disqus.com/?ref_noscript">comments powered by Disqus.</a></noscript>
</div>
</div>
</div>
</div>
</div>
<footer class="footer">
<div class="container">
<div class="row">
<div class="col-sm-11">
<p class="text-center">
Nemo 2023 |
Powered by <a href="https://github.com/aksakalli/jekyll-doc-theme">Jekyll Doc Theme</a>
</p>
<br>
<p class="text-muted text-center">
Disclaimer: Apache Nemo (incubating) is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.
<br>
<br>
Copyright © 2022 The Apache Software Foundation. Apache and the Apache feather logo are trademarks of The Apache Software Foundation
</p>
<!-- <p class="text-muted">Place sticky footer content here.</p> -->
</div>
<div class="col-sm-1">
<a href="https://www.apache.org/events/current-event.html">
<img src="https://www.apache.org/events/current-event-125x125.png">
</a>
</div>
</div>
</div>
</footer>
<script>
var baseurl = ''
</script>
<script src="/assets/js/jquery-1.12.4.min.js "></script>
<script src="/assets/js/bootstrap.min.js "></script>
<script src="/assets/js/typeahead.bundle.min.js "></script>
<script src="/assets/js/main.js "></script>
</body>
</html>