blob: bdfb4b1105578e47bc0fb274cc278b922e900b98 [file] [log] [blame]
<!DOCTYPE html>
<html lang="en-US">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="/assets/css/custom.css">
<link rel="stylesheet" href="/assets/css/main.css">
<link rel="stylesheet" href="/assets/css/font-awesome.min.css">
<link rel="shortcut icon" href="/favicon.ico?1">
<!-- Begin Jekyll SEO tag v2.8.0 -->
<title>Overview | Nemo</title>
<meta name="generator" content="Jekyll v3.9.3" />
<meta property="og:title" content="Overview" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="Nemo aims to optimize data processing for better performance and datacenter efficiency, not only in general and common conditions, but also with various deployment characteristics. Such characteristics include processing data on specific resource environments, like transient resources, and running jobs with specific attributes, like skewed data." />
<meta property="og:description" content="Nemo aims to optimize data processing for better performance and datacenter efficiency, not only in general and common conditions, but also with various deployment characteristics. Such characteristics include processing data on specific resource environments, like transient resources, and running jobs with specific attributes, like skewed data." />
<link rel="canonical" href="http://nemo.apache.org//docs/home/" />
<meta property="og:url" content="http://nemo.apache.org//docs/home/" />
<meta property="og:site_name" content="Nemo" />
<meta property="og:type" content="article" />
<meta property="article:published_time" content="2023-05-11T14:14:28+09:00" />
<meta name="twitter:card" content="summary" />
<meta property="twitter:title" content="Overview" />
<script type="application/ld+json">
{"@context":"https://schema.org","@type":"WebPage","dateModified":"2023-05-11T14:14:28+09:00","datePublished":"2023-05-11T14:14:28+09:00","description":"Nemo aims to optimize data processing for better performance and datacenter efficiency, not only in general and common conditions, but also with various deployment characteristics. Such characteristics include processing data on specific resource environments, like transient resources, and running jobs with specific attributes, like skewed data.","headline":"Overview","url":"http://nemo.apache.org//docs/home/"}</script>
<!-- End Jekyll SEO tag -->
<link rel="canonical" href="http://nemo.apache.org//docs/home/">
<link rel="alternate" type="application/rss+xml" title="Nemo" href="http://nemo.apache.org//feed.xml" />
</head>
<body>
<nav class="navbar navbar-default navbar-fixed-top">
<div class="container navbar-container">
<div class="navbar-header">
<button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
<span class="sr-only">Toggle navigation</span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
<a class="navbar-brand" href="/">
<span><img src="/assets/img/nemo-logo.png" alt="Logo"></span>
</a>
</div>
<div id="navbar" class="collapse navbar-collapse">
<ul class="nav navbar-nav">
<li class="active" ><a href="/docs/home/">Docs</a></li>
<li ><a href="/apidocs">APIs</a></li>
<li ><a href="/pages/downloads">Downloads</a></li>
<li ><a href="/pages/talks">Talks</a></li>
<li ><a href="/pages/team">Team</a></li>
<li ><a href="/pages/license">License</a></li>
<li ><a href="/blog/2022/09/10/release-note-0.4/">Blog</a></li>
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">ASF<span class="caret"></span></a>
<ul class="dropdown-menu">
<li><a href="http://www.apache.org/">Apache Software Foundation</a></li>
<li><a href="https://www.apache.org/foundation/how-it-works.html">How Apache Works</a></li>
<li><a href="https://www.apache.org/licenses/">License</a></li>
<li><a href="https://www.apache.org/security/">Security</a></li>
<li><a href="https://privacy.apache.org/policies/privacy-policy-public.html">Privacy</a></li>
<li><a href="https://www.apache.org/foundation/sponsorship.html">Sponsoring Apache</a></li>
<li><a href="https://www.apache.org/foundation/thanks.html">Thanks</a></li>
</ul>
</li>
</ul>
<div class="navbar-right">
<form class="navbar-form navbar-left">
<div class="form-group has-feedback">
<input id="search-box" type="search" class="form-control" placeholder="Search...">
<i class="fa fa-search form-control-feedback"></i>
</div>
</form>
<ul class="nav navbar-nav">
<li><a href="https://github.com/apache/incubator-nemo"><i class="fa fa-github" aria-hidden="true"></i></a></li>
</ul>
</div>
</div>
</div>
</nav>
<div class="page-content">
<div class="wrapper">
<div class="container">
<div class="row">
<div class="col-md-4">
<div class="panel-group" id="accordion" role="tablist" aria-multiselectable="true">
<div class="panel panel-default">
<div class="panel-heading">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapse-1" aria-expanded="false" aria-controls="collapse-1">
Getting Started
</a>
</h4>
</div>
<div class="panel-collapse" id="collapse-1" role="tabpanel" aria-label="Side Navigation">
<div class="list-group">
<a class="list-group-item active" href="/docs/home/">Overview</a>
<a class="list-group-item " href="/docs/getting_started/">Getting Started</a>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapse-2" aria-expanded="false" aria-controls="collapse-2">
Optimizations
</a>
</h4>
</div>
<div class="panel-collapse collapse" id="collapse-2" role="tabpanel" aria-label="Side Navigation">
<div class="list-group">
<a class="list-group-item " href="/docs/ir/">Nemo Intermediate Representation (IR)</a>
<a class="list-group-item " href="/docs/passes_and_policies/">Passes and Policies</a>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapse-3" aria-expanded="false" aria-controls="collapse-3">
System Designs
</a>
</h4>
</div>
<div class="panel-collapse collapse" id="collapse-3" role="tabpanel" aria-label="Side Navigation">
<div class="list-group">
<a class="list-group-item " href="/docs/compiler_design/">Compiler Design</a>
<a class="list-group-item " href="/docs/runtime_design/">Runtime Design</a>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapse-4" aria-expanded="false" aria-controls="collapse-4">
Contribute
</a>
</h4>
</div>
<div class="panel-collapse collapse" id="collapse-4" role="tabpanel" aria-label="Side Navigation">
<div class="list-group">
<a class="list-group-item " href="/docs/contribute/">Contribute</a>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapse-5" aria-expanded="false" aria-controls="collapse-5">
Security
</a>
</h4>
</div>
<div class="panel-collapse collapse" id="collapse-5" role="tabpanel" aria-label="Side Navigation">
<div class="list-group">
<a class="list-group-item " href="/docs/security/">Security Guide</a>
</div>
</div>
</div>
</div>
</div>
<div class="col-md-8">
<h1>Overview</h1>
<div id="markdown-content-container"><p>Nemo aims to optimize data processing for better performance and datacenter efficiency, not only in general and common conditions, but also with various <em>deployment characteristics</em>.
Such characteristics include processing data on <em>specific resource environments</em>, like transient resources, and running <em>jobs with specific attributes</em>, like skewed data.</p>
<p>There exists many data processing systems with different designs to solve each of such problems it targets, but it fails to cover or adapt to unconsidered cases without substantial effort for modification.
The primary reason is because system runtime behaviors are hidden and planted inside the system core to hide the complexity of distributed computing.
This makes it very hard for a single system to support different <em>deployment characteristics</em> with different <em>runtime behaviors</em> without substantial effort.</p>
<p>To solve this problem and easily modify <em>runtime behaviors</em> for different <em>deployment characteristics</em>, Nemo expresses workloads using the <a href="../ir">Nemo Intermediate Representation (IR)</a>, which represents the logical notion of data processing applications and its runtime behaviors on separate layers.
These layers can be easily modified through a set of high-level <a href="../passes_and_policies">graph pass</a> interfaces, exposed by the <a href="../compiler_design">Nemo Compiler</a>, enabling users to flexibly modify <em>runtime behaviors</em> at both compile-time and runtime.
Works represented this way can be executed by the <a href="../runtime_design">Nemo Execution Runtime</a> through its <a href="../extending_Nemo">modular and extensible</a> design.</p>
<p><img src="/assets/img/overview.jpg" alt="image" /></p>
<p>If you want to contribute, please refer to the <a href="../contribute">contribute</a> page. Otherwise, hit the button below to get started with Nemo!</p>
<p><br /></p>
<div class="text-center">
<a href="../getting_started/" class="btn btn-default btn-lg">Get Started!</a>
</div>
</div>
<div style="clear:both;">
<p class="text-center">
<br />
<a target="_blank" href="https://github.com/apache/incubator-nemo-website/blob/asf-site/_docs/index.md" class="btn btn-default" role="button">
<i class="fa fa-pencil fa-lg"></i> Improve this page
</a>
</p>
</div>
<hr>
<ul class="pager">
<li class="previous disabled">
<a>
<span aria-hidden="true">&larr;</span> Previous
</a>
</li>
<li class="next">
<a href="/docs/getting_started/">
Next <span aria-hidden="true">&rarr;</span>
</a>
</li>
</ul>
<div class="clear"></div>
<div id="disqus_thread"></div>
<script>
/**
* RECOMMENDED CONFIGURATION VARIABLES: EDIT AND UNCOMMENT THE SECTION BELOW TO INSERT DYNAMIC VALUES FROM YOUR PLATFORM OR CMS.
* LEARN WHY DEFINING THESE VARIABLES IS IMPORTANT: https://disqus.com/admin/universalcode/#configuration-variables */
/*
var disqus_config = function () {
this.page.url = PAGE_URL; // Replace PAGE_URL with your page's canonical URL variable
this.page.identifier = PAGE_IDENTIFIER; // Replace PAGE_IDENTIFIER with your page's unique identifier variable
};
*/
(function() { // DON'T EDIT BELOW THIS LINE
var d = document, s = d.createElement('script');
s.src = 'https://apache-nemo.disqus.com/embed.js';
s.setAttribute('data-timestamp', +new Date());
(d.head || d.body).appendChild(s);
})();
</script>
<noscript>Please enable JavaScript to view the <a href="https://disqus.com/?ref_noscript">comments powered by Disqus.</a></noscript>
</div>
</div>
</div>
</div>
</div>
<footer class="footer">
<div class="container">
<div class="row">
<div class="col-sm-11">
<p class="text-center">
Nemo 2023 |
Powered by <a href="https://github.com/aksakalli/jekyll-doc-theme">Jekyll Doc Theme</a>
</p>
<br>
<p class="text-muted text-center">
Disclaimer: Apache Nemo (incubating) is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.
<br>
<br>
Copyright © 2022 The Apache Software Foundation. Apache and the Apache feather logo are trademarks of The Apache Software Foundation
</p>
<!-- <p class="text-muted">Place sticky footer content here.</p> -->
</div>
<div class="col-sm-1">
<a href="https://www.apache.org/events/current-event.html">
<img src="https://www.apache.org/events/current-event-125x125.png">
</a>
</div>
</div>
</div>
</footer>
<script>
var baseurl = ''
</script>
<script src="/assets/js/jquery-1.12.4.min.js "></script>
<script src="/assets/js/bootstrap.min.js "></script>
<script src="/assets/js/typeahead.bundle.min.js "></script>
<script src="/assets/js/main.js "></script>
</body>
</html>