blob: 2da9e5d5260ee4f2e2384301602c5b0575f1dc90 [file] [log] [blame]
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
<title>Apache Flink: Stateful Functions 2.0 - An Event-driven Database on Apache Flink</title>
<link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
<link rel="icon" href="/favicon.ico" type="image/x-icon">
<!-- Bootstrap -->
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.4.1/css/bootstrap.min.css">
<link rel="stylesheet" href="/css/flink.css">
<link rel="stylesheet" href="/css/syntax.css">
<!-- Blog RSS feed -->
<link href="/blog/feed.xml" rel="alternate" type="application/rss+xml" title="Apache Flink Blog: RSS feed" />
<!-- jQuery (necessary for Bootstrap's JavaScript plugins) -->
<!-- We need to load Jquery in the header for custom google analytics event tracking-->
<script src="/js/jquery.min.js"></script>
<!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
<!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
<!--[if lt IE 9]>
<script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
<![endif]-->
</head>
<body>
<!-- Main content. -->
<div class="container">
<div class="row">
<div id="sidebar" class="col-sm-3">
<!-- Top navbar. -->
<nav class="navbar navbar-default">
<!-- The logo. -->
<div class="navbar-header">
<button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
<div class="navbar-logo">
<a href="/">
<img alt="Apache Flink" src="/img/flink-header-logo.svg" width="147px" height="73px">
</a>
</div>
</div><!-- /.navbar-header -->
<!-- The navigation links. -->
<div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
<ul class="nav navbar-nav navbar-main">
<!-- First menu section explains visitors what Flink is -->
<!-- What is Stream Processing? -->
<!--
<li><a href="/streamprocessing1.html">What is Stream Processing?</a></li>
-->
<!-- What is Flink? -->
<li><a href="/flink-architecture.html">What is Apache Flink?</a></li>
<!-- What is Stateful Functions? -->
<li><a href="/stateful-functions.html">What is Stateful Functions?</a></li>
<!-- Use cases -->
<li><a href="/usecases.html">Use Cases</a></li>
<!-- Powered by -->
<li><a href="/poweredby.html">Powered By</a></li>
&nbsp;
<!-- Second menu section aims to support Flink users -->
<!-- Downloads -->
<li><a href="/downloads.html">Downloads</a></li>
<!-- Getting Started -->
<li class="dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" href="#">Getting Started<span class="caret"></span></a>
<ul class="dropdown-menu">
<li><a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/getting-started/index.html" target="_blank">With Flink <small><span class="glyphicon glyphicon-new-window"></span></small></a></li>
<li><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-release-2.1/getting-started/project-setup.html" target="_blank">With Flink Stateful Functions <small><span class="glyphicon glyphicon-new-window"></span></small></a></li>
<li><a href="/training.html">Training Course</a></li>
</ul>
</li>
<!-- Documentation -->
<li class="dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" href="#">Documentation<span class="caret"></span></a>
<ul class="dropdown-menu">
<li><a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11" target="_blank">Flink 1.11 (Latest stable release) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li>
<li><a href="https://ci.apache.org/projects/flink/flink-docs-master" target="_blank">Flink Master (Latest Snapshot) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li>
<li><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-release-2.1" target="_blank">Flink Stateful Functions 2.1 (Latest stable release) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li>
<li><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-master" target="_blank">Flink Stateful Functions Master (Latest Snapshot) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li>
</ul>
</li>
<!-- getting help -->
<li><a href="/gettinghelp.html">Getting Help</a></li>
<!-- Blog -->
<li class="active"><a href="/blog/"><b>Flink Blog</b></a></li>
<!-- Flink-packages -->
<li>
<a href="https://flink-packages.org" target="_blank">flink-packages.org <small><span class="glyphicon glyphicon-new-window"></span></small></a>
</li>
&nbsp;
<!-- Third menu section aim to support community and contributors -->
<!-- Community -->
<li><a href="/community.html">Community &amp; Project Info</a></li>
<!-- Roadmap -->
<li><a href="/roadmap.html">Roadmap</a></li>
<!-- Contribute -->
<li><a href="/contributing/how-to-contribute.html">How to Contribute</a></li>
<!-- GitHub -->
<li>
<a href="https://github.com/apache/flink" target="_blank">Flink on GitHub <small><span class="glyphicon glyphicon-new-window"></span></small></a>
</li>
&nbsp;
<!-- Language Switcher -->
<li>
<!-- link to the Chinese home page when current is blog page -->
<a href="/zh">中文版</a>
</li>
</ul>
<ul class="nav navbar-nav navbar-bottom">
<hr />
<!-- Twitter -->
<li><a href="https://twitter.com/apacheflink" target="_blank">@ApacheFlink <small><span class="glyphicon glyphicon-new-window"></span></small></a></li>
<!-- Visualizer -->
<li class=" hidden-md hidden-sm"><a href="/visualizer/" target="_blank">Plan Visualizer <small><span class="glyphicon glyphicon-new-window"></span></small></a></li>
<hr />
<li><a href="https://apache.org" target="_blank">Apache Software Foundation <small><span class="glyphicon glyphicon-new-window"></span></small></a></li>
<li>
<style>
.smalllinks:link {
display: inline-block !important; background: none; padding-top: 0px; padding-bottom: 0px; padding-right: 0px; min-width: 75px;
}
</style>
<a class="smalllinks" href="https://www.apache.org/licenses/" target="_blank">License</a> <small><span class="glyphicon glyphicon-new-window"></span></small>
<a class="smalllinks" href="https://www.apache.org/security/" target="_blank">Security</a> <small><span class="glyphicon glyphicon-new-window"></span></small>
<a class="smalllinks" href="https://www.apache.org/foundation/sponsorship.html" target="_blank">Donate</a> <small><span class="glyphicon glyphicon-new-window"></span></small>
<a class="smalllinks" href="https://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a> <small><span class="glyphicon glyphicon-new-window"></span></small>
</li>
</ul>
</div><!-- /.navbar-collapse -->
</nav>
</div>
<div class="col-sm-9">
<div class="row-fluid">
<div class="col-sm-12">
<div class="row">
<h1>Stateful Functions 2.0 - An Event-driven Database on Apache Flink</h1>
<p><i>Making the Stream Processor for Event-driven Applications what the Database was to CRUD Applications</i></p>
<article>
<p>07 Apr 2020 Stephan Ewen (<a href="https://twitter.com/stephanewen">@stephanewen</a>)</p>
<p>Today, we are announcing the release of Stateful Functions (StateFun) 2.0 — the first release of Stateful Functions as part of the Apache Flink project.
This release marks a big milestone: Stateful Functions 2.0 is not only an API update, but the <strong>first version of an event-driven database</strong> that is built on Apache Flink.</p>
<p>Stateful Functions 2.0 makes it possible to combine StateFun’s powerful approach to state and composition with the elasticity, rapid scaling/scale-to-zero and rolling upgrade capabilities of FaaS implementations like AWS Lambda and modern resource orchestration frameworks like Kubernetes.</p>
<p>With these features, Stateful Functions 2.0 addresses <a href="https://www2.eecs.berkeley.edu/Pubs/TechRpts/2019/EECS-2019-3.pdf">two of the most cited shortcomings</a> of many FaaS setups today: consistent state and efficient messaging between functions.</p>
<div class="page-toc">
<ul id="markdown-toc">
<li><a href="#an-event-driven-database" id="markdown-toc-an-event-driven-database">An Event-driven Database</a></li>
<li><a href="#event-driven-database-vs-requestresponse-database" id="markdown-toc-event-driven-database-vs-requestresponse-database">“Event-driven Database” vs. “Request/Response Database”</a> <ul>
<li><a href="#state-and-consistency" id="markdown-toc-state-and-consistency">State and Consistency</a></li>
</ul>
</li>
<li><a href="#remote-co-located-or-embedded-functions" id="markdown-toc-remote-co-located-or-embedded-functions">Remote, Co-located or Embedded Functions</a> <ul>
<li><a href="#remote-functions" id="markdown-toc-remote-functions">Remote Functions</a></li>
<li><a href="#co-located-functions" id="markdown-toc-co-located-functions">Co-located Functions</a></li>
<li><a href="#embedded-functions" id="markdown-toc-embedded-functions">Embedded Functions</a></li>
</ul>
</li>
<li><a href="#loading-data-into-the-database" id="markdown-toc-loading-data-into-the-database">Loading Data into the Database</a></li>
<li><a href="#try-it-out-and-get-involved" id="markdown-toc-try-it-out-and-get-involved">Try it out and get involved!</a></li>
<li><a href="#thank-you" id="markdown-toc-thank-you">Thank you!</a></li>
</ul>
</div>
<h2 id="an-event-driven-database">An Event-driven Database</h2>
<p>When Stateful Functions joined Apache Flink at the beginning of this year, the project had started as a library on top of Flink to build general-purpose event-driven applications. Users would implement <em>functions</em> that receive and send messages, and maintain state in persistent variables. Flink provided the runtime with efficient exactly-once state and messaging. Stateful Functions 1.0 was a FaaS-inspired mix between stream processing and actor programming — on steroids.</p>
<div style="line-height:60%;">
<br />
</div>
<center>
<figure>
<img src="/img/blog/2020-04-07-release-statefun-2.0.0/image2.png" width="600px" alt="Statefun 1" />
<br /><br />
<figcaption><i><b>Fig.1:</b> A ride-sharing app as a Stateful Functions example.</i></figcaption>
</figure>
</center>
<div style="line-height:150%;">
<br />
</div>
<p>In version 2.0, Stateful Functions now physically decouples the functions from Flink and the JVM, to invoke them through simple services. That makes it possible to execute functions on a FaaS platform, a Kubernetes deployment or behind a (micro) service.</p>
<p>Flink invokes the functions through a service endpoint via HTTP or gRPC based on incoming events, and supplies state access. The system makes sure that only one invocation per entity (<code>type</code>+<code>ID</code>) is ongoing at any point in time, thus guaranteeing consistency through isolation.
By supplying state access as part of the function invocation, the functions themselves behave like stateless applications and can be managed with the same simplicity and benefits: rapid scalability, scale-to-zero, rolling/zero-downtime upgrades and so on.</p>
<center>
<figure>
<img src="/img/blog/2020-04-07-release-statefun-2.0.0/image5.png" width="600px" alt="Statefun 2" />
<br /><br />
<figcaption><i><b>Fig.2:</b> In Stateful Functions 2.0, functions are stateless and state access is part of the function invocation.</i></figcaption>
</figure>
</center>
<div style="line-height:150%;">
<br />
</div>
<p>The functions can be implemented in any programming language that can handle HTTP requests or bring up a gRPC server. The <a href="https://github.com/apache/flink-statefun">StateFun project</a> includes a very slim SDK for Python, taking requests and dispatching them to annotated functions. We aim to provide similar SDKs for other languages, such as Go, JavaScript or Rust. Users do not need to write any Flink code (or JVM code) at all; data ingresses/egresses and function endpoints can be defined in a compact YAML spec.</p>
<div style="line-height:60%;">
<br />
</div>
<div class="row">
<div class="col-lg-6">
<div class="text-center">
<figure>
<img src="/img/blog/2020-04-07-release-statefun-2.0.0/image3.png" width="600px" alt="Statefun 3" />
<br /><br />
<figcaption><i><b>Fig.3:</b> A module declaring a remote endpoint and a function type.</i></figcaption>
</figure>
</div>
</div>
<div class="col-lg-6">
<div class="text-center">
<figure>
<div style="line-height:540%;">
<br />
</div>
<img src="/img/blog/2020-04-07-release-statefun-2.0.0/image10.png" width="600px" alt="Statefun 4" />
<br /><br />
<figcaption><i><b>Fig.4:</b> A Python implementation of a simple classifier function.</i></figcaption>
</figure>
</div>
</div>
</div>
<div style="line-height:150%;">
<br />
</div>
<p>The Flink processes (and the JVM) are not executing any user-code at all — though this is possible, for performance reasons (see <a href="#embedded-functions">Embedded Functions</a>). Rather than running application-specific dataflows, Flink here stores the state of the functions and provides the dynamic messaging plane through which functions message each other, carefully dispatching messages/invocations to the event-driven functions/services to maintain consistency guarantees.</p>
<blockquote>
<p><em>Effectively, Flink takes the role of the database, but tailored towards event-driven functions and services.
It integrates state storage with the messaging between (and the invocations of) functions and services.
Because of this, Stateful Functions 2.0 can be thought of as an “Event-driven Database” on Apache Flink.</em></p>
</blockquote>
<h2 id="event-driven-database-vs-requestresponse-database">“Event-driven Database” vs. “Request/Response Database”</h2>
<p>In the case of a traditional database or key/value store (let’s call them request/response databases), the application issues queries to the database (e.g. SQL via JDBC, GET/PUT via HTTP). In contrast, an event-driven database like StateFun <strong><em>inverts</em></strong> that relationship between database and application: the database invokes the functions/services based on arriving messages. This fits very naturally with FaaS and many event-driven application architectures.</p>
<div style="line-height:60%;">
<br />
</div>
<center>
<figure>
<img src="/img/blog/2020-04-07-release-statefun-2.0.0/image7.png" width="600px" alt="Statefun 5" />
<br /><br />
<figcaption><i><b>Fig.5:</b> Stateful Functions 2.0 inverts the relationship between database and application.</i></figcaption>
</figure>
</center>
<div style="line-height:150%;">
<br />
</div>
<p>In the case of applications built on request/response databases, the database is responsible only for the state. Communication between different functions/services is a separate concern handled within the application layer. In contrast to that, an event-driven database takes care of both state storage and message transport, in a tightly integrated manner.</p>
<p>Similar to <a href="https://www.brianstorti.com/the-actor-model/">Actor Programming</a>, Stateful Functions uses the idea of <em>addressable entities</em> - here, the entity is a function <code>type</code> with an invocation scoped to an <code>ID</code>. These addressable entities own the state and are the targets of messages. Different to actor systems is that the application logic is external and the addressable entities are not physical objects in memory (i.e. actors), but rows in Flink’s managed state, together with the entities’ mailboxes.</p>
<h3 id="state-and-consistency">State and Consistency</h3>
<p>Besides matching the needs of serverless applications and FaaS well, the event-driven database approach also helps with simplifying consistent state management.</p>
<p>Consider the example below, with two entities of an application — for example two microservices (<em>Service 1</em>, <em>Service 2</em>). <em>Service 1</em> is invoked, updates the state in the database, and sends a request to <em>Service 2</em>. Assume that this request fails. There is, in general, no way for <em>Service 1</em> to know whether <em>Service 2</em> processed the request and updated its state or not (c.f. <a href="https://en.wikipedia.org/wiki/Two_Generals%27_Problem">Two Generals Problem</a>). To work around that, many techniques exist — making requests idempotent and retrying, commit/rollback protocols, or external transaction coordinators, for example. Solving this in the application layer is complex enough, and including the database into these approaches only adds more complexity.</p>
<p>In the scenario where the event-driven database takes care of state and messaging, we have a much easier problem to solve. Assume one shard of the database receives the initial message, updates its state, invokes <em>Service 1</em>, and routes the message produced by the function to another shard, to be delivered to <em>Service 2</em>. Now assume message transport errored — it may have failed or not, we cannot know for certain. Because the database is in charge of state and messaging, it can offer a generic solution to make sure that either both go through or none does, for example through transactions or <a href="https://dl.acm.org/doi/abs/10.14778/3137765.3137777">consistent snapshots</a>. The application functions are stateless and their invocations without side effects, which means they can be re-invoked again without implications on consistency.</p>
<div style="line-height:60%;">
<br />
</div>
<center>
<figure>
<img src="/img/blog/2020-04-07-release-statefun-2.0.0/image8.png" width="600px" alt="Statefun 6" />
<br /><br />
<figcaption><i><b>Fig.6:</b> The event-driven database integrates state access and messaging, guaranteeing consistency.</i></figcaption>
</figure>
</center>
<div style="line-height:150%;">
<br />
</div>
<p>That is the big lesson we learned from working on stream processing technology in the past years: <strong>state access/updates and messaging need to be integrated</strong>. This gives you consistency, scalable behavior and backpressures well based on both state access and compute bottlenecks.</p>
<p>Despite state and computation being physically separated here, the scheduling/dispatching of function invocations is still integrated and physically co-located with state access, preserving the consistency guarantees given by physical state/compute co-location.</p>
<h2 id="remote-co-located-or-embedded-functions">Remote, Co-located or Embedded Functions</h2>
<p>Functions can be deployed in various ways that trade off loose coupling and independent scaling with performance overhead. Each module of functions can be of a different kind, so some functions can run remote, while others could run embedded.</p>
<h3 id="remote-functions">Remote Functions</h3>
<p><em>Remote Functions</em> are the mechanism described so far, where functions are deployed separately from the Flink StateFun cluster. The state/messaging tier (i.e. the Flink processes) and the function tier can be deployed and scaled independently. All function invocations are remote and have to go through the endpoint service.</p>
<div style="line-height:60%;">
<br />
</div>
<center>
<img src="/img/blog/2020-04-07-release-statefun-2.0.0/image6.png" width="600px" alt="Statefun 7" />
</center>
<div style="line-height:150%;">
<br />
</div>
<p>In a similar way as databases are accessed via a standardized protocol (e.g. ODBC/JDBC for relational databases, REST for many key/value stores), StateFun 2.0 invokes functions and services through a standardized protocol: HTTP or gRPC with data in a well-defined ProtoBuf schema.</p>
<h3 id="co-located-functions">Co-located Functions</h3>
<p>An alternative way of deploying functions is <em>co-location</em> with the Flink JVM processes. In such a setup, each Flink TaskManager would talk to one function process sitting “next to it”. A common way to do this is to use a system like Kubernetes and deploy pods consisting of a Flink container and the function container that communicate via the pod-local network.</p>
<p>This mode supports different languages while avoiding to route invocations through a Service/Gateway/LoadBalancer, but it cannot scale the state and compute parts independently.</p>
<div style="line-height:60%;">
<br />
</div>
<center>
<img src="/img/blog/2020-04-07-release-statefun-2.0.0/image9.png" width="600px" alt="Statefun 8" />
</center>
<div style="line-height:150%;">
<br />
</div>
<p>This style of deployment is similar to how <a href="https://beam.apache.org/roadmap/portability/">Apache Beam’s portability layer</a> and <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/tutorials/python_table_api.html">Flink’s Python API</a> deploy their non-JVM language SDKs.</p>
<h3 id="embedded-functions">Embedded Functions</h3>
<p><em>Embedded Functions</em> are the mode of Stateful Functions 1.0 and Flink’s Java/Scala stream processing APIs. Functions are deployed into the JVM and are directly invoked with the messages and state access. This is the most performant way, though at the cost of only supporting JVM languages.</p>
<div style="line-height:60%;">
<br />
</div>
<center>
<img src="/img/blog/2020-04-07-release-statefun-2.0.0/image11.png" width="600px" alt="Statefun 9" />
</center>
<div style="line-height:150%;">
<br />
</div>
<p>Following the database analogy, embedded functions are a bit like <em>stored procedures</em>, but in a principled way: the functions here are normal Java/Scala/Kotlin functions implementing standard interfaces and can be developed or tested in any IDE.</p>
<h2 id="loading-data-into-the-database">Loading Data into the Database</h2>
<p>When building a new stateful application, you usually don’t start from a completely blank slate. Often, the application has initial state, such as initial “bootstrap” state, or state from previous versions of the application. When using a database, one could simply bulk load the data to prepare the application.</p>
<p>The equivalent step for Flink would be to write a <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/ops/state/savepoints.html">savepoint</a> that contains the initial state. Savepoints are snapshots of the state of the distributed stream processing application and can be passed to Flink to start processing from that state. Think of them as a database dump, but of a distributed streaming database. In the case of StateFun, the savepoint would contain the state of the functions.</p>
<p>To create a savepoint for a Stateful Functions program, check out the <a href="https://ci.apache.org/projects/flink/flink-statefun-docs-release-2.0/deployment-and-operations/state-bootstrap.html">State Bootstrapping API</a> that is part of StateFun 2.0. The State Bootstrapping API uses Flink’s <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/dev/batch/">DataSet API</a>, but we plan to expand this to use SQL in the next versions.</p>
<h2 id="try-it-out-and-get-involved">Try it out and get involved!</h2>
<p>We hope that we could convey some of the excitement we feel about Stateful Functions. If we managed to pique your curiosity, try it out — for example, starting with <a href="https://ci.apache.org/projects/flink/flink-statefun-docs-release-2.0/getting-started/python_walkthrough.html">this walkthrough</a>.</p>
<p>The project is still in a comparatively early stage, so if you want to get involved, there is lots to work on: SDKs for other languages (e.g. Go, JavaScript, Rust), ingresses/egresses and tools for testing, among others.</p>
<p>To follow the project and learn more, please check out these resources:</p>
<ul>
<li>Code: <a href="https://github.com/apache/flink-statefun">https://github.com/apache/flink-statefun</a></li>
<li>Docs: <a href="https://ci.apache.org/projects/flink/flink-statefun-docs-release-2.0/">https://ci.apache.org/projects/flink/flink-statefun-docs-release-2.0/</a></li>
<li>Apache Flink project site: <a href="https://flink.apache.org/">https://flink.apache.org/</a></li>
<li>Apache Flink on Twitter: <a href="https://twitter.com/apacheflink">@ApacheFlink</a></li>
<li>Stateful Functions Webpage: <a href="https://statefun.io">https://statefun.io</a></li>
<li>Stateful Functions on Twitter: <a href="https://twitter.com/statefun_io">@StateFun_IO</a></li>
</ul>
<h2 id="thank-you">Thank you!</h2>
<p>The Apache Flink community would like to thank all contributors that have made this release possible:</p>
<p>David Anderson, Dian Fu, Igal Shilman, Seth Wiesman, Stephan Ewen, Tzu-Li (Gordon) Tai, hequn8128</p>
</article>
</div>
<div class="row">
<div id="disqus_thread"></div>
<script type="text/javascript">
/* * * CONFIGURATION VARIABLES: EDIT BEFORE PASTING INTO YOUR WEBPAGE * * */
var disqus_shortname = 'stratosphere-eu'; // required: replace example with your forum shortname
/* * * DON'T EDIT BELOW THIS LINE * * */
(function() {
var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true;
dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js';
(document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq);
})();
</script>
</div>
</div>
</div>
</div>
</div>
<hr />
<div class="row">
<div class="footer text-center col-sm-12">
<p>Copyright © 2014-2019 <a href="http://apache.org">The Apache Software Foundation</a>. All Rights Reserved.</p>
<p>Apache Flink, Flink®, Apache®, the squirrel logo, and the Apache feather logo are either registered trademarks or trademarks of The Apache Software Foundation.</p>
<p><a href="/privacy-policy.html">Privacy Policy</a> &middot; <a href="/blog/feed.xml">RSS feed</a></p>
</div>
</div>
</div><!-- /.container -->
<!-- Include all compiled plugins (below), or include individual files as needed -->
<script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery.matchHeight/0.7.0/jquery.matchHeight-min.js"></script>
<script src="/js/codetabs.js"></script>
<script src="/js/stickysidebar.js"></script>
<!-- Google Analytics -->
<script>
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
})(window,document,'script','//www.google-analytics.com/analytics.js','ga');
ga('create', 'UA-52545728-1', 'auto');
ga('send', 'pageview');
</script>
</body>
</html>