| <!DOCTYPE html> |
| <html lang="en"> |
| <head> |
| <meta charset="utf-8"> |
| <meta http-equiv="X-UA-Compatible" content="IE=edge"> |
| <meta name="viewport" content="width=device-width, initial-scale=1"> |
| <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> |
| <title>Apache Flink: Stateful Functions — Event-driven Applications on Apache Flink</title> |
| <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon"> |
| <link rel="icon" href="/favicon.ico" type="image/x-icon"> |
| |
| <!-- Bootstrap --> |
| <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.4.1/css/bootstrap.min.css"> |
| <link rel="stylesheet" href="/css/flink.css"> |
| <link rel="stylesheet" href="/css/syntax.css"> |
| |
| <!-- Blog RSS feed --> |
| <link href="/blog/feed.xml" rel="alternate" type="application/rss+xml" title="Apache Flink Blog: RSS feed" /> |
| |
| <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> |
| <!-- We need to load Jquery in the header for custom google analytics event tracking--> |
| <script src="/js/jquery.min.js"></script> |
| |
| <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> |
| <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> |
| <!--[if lt IE 9]> |
| <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script> |
| <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script> |
| <![endif]--> |
| </head> |
| <body> |
| |
| |
| <!-- Main content. --> |
| <div class="container"> |
| <div class="row"> |
| |
| |
| <div id="sidebar" class="col-sm-3"> |
| |
| |
| <!-- Top navbar. --> |
| <nav class="navbar navbar-default"> |
| <!-- The logo. --> |
| <div class="navbar-header"> |
| <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> |
| <span class="icon-bar"></span> |
| <span class="icon-bar"></span> |
| <span class="icon-bar"></span> |
| </button> |
| <div class="navbar-logo"> |
| <a href="/"> |
| <img alt="Apache Flink" src="/img/flink-header-logo.svg" width="147px" height="73px"> |
| </a> |
| </div> |
| </div><!-- /.navbar-header --> |
| |
| <!-- The navigation links. --> |
| <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> |
| <ul class="nav navbar-nav navbar-main"> |
| |
| <!-- First menu section explains visitors what Flink is --> |
| |
| <!-- What is Stream Processing? --> |
| <!-- |
| <li><a href="/streamprocessing1.html">What is Stream Processing?</a></li> |
| --> |
| |
| <!-- What is Flink? --> |
| <li><a href="/flink-architecture.html">What is Apache Flink?</a></li> |
| |
| |
| |
| <!-- What is Stateful Functions? --> |
| |
| <li class="active"><a href="/stateful-functions.html">What is Stateful Functions?</a></li> |
| |
| <!-- Use cases --> |
| <li><a href="/usecases.html">Use Cases</a></li> |
| |
| <!-- Powered by --> |
| <li><a href="/poweredby.html">Powered By</a></li> |
| |
| |
| |
| <!-- Second menu section aims to support Flink users --> |
| |
| <!-- Downloads --> |
| <li><a href="/downloads.html">Downloads</a></li> |
| |
| <!-- Getting Started --> |
| <li class="dropdown"> |
| <a class="dropdown-toggle" data-toggle="dropdown" href="#">Getting Started<span class="caret"></span></a> |
| <ul class="dropdown-menu"> |
| <li><a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/getting-started/index.html" target="_blank">With Flink <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-release-2.1/getting-started/project-setup.html" target="_blank">With Flink Stateful Functions <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="/training.html">Training Course</a></li> |
| </ul> |
| </li> |
| |
| <!-- Documentation --> |
| <li class="dropdown"> |
| <a class="dropdown-toggle" data-toggle="dropdown" href="#">Documentation<span class="caret"></span></a> |
| <ul class="dropdown-menu"> |
| <li><a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11" target="_blank">Flink 1.11 (Latest stable release) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="https://ci.apache.org/projects/flink/flink-docs-master" target="_blank">Flink Master (Latest Snapshot) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-release-2.1" target="_blank">Flink Stateful Functions 2.1 (Latest stable release) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-master" target="_blank">Flink Stateful Functions Master (Latest Snapshot) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| </ul> |
| </li> |
| |
| <!-- getting help --> |
| <li><a href="/gettinghelp.html">Getting Help</a></li> |
| |
| <!-- Blog --> |
| <li><a href="/blog/"><b>Flink Blog</b></a></li> |
| |
| |
| <!-- Flink-packages --> |
| <li> |
| <a href="https://flink-packages.org" target="_blank">flink-packages.org <small><span class="glyphicon glyphicon-new-window"></span></small></a> |
| </li> |
| |
| |
| <!-- Third menu section aim to support community and contributors --> |
| |
| <!-- Community --> |
| <li><a href="/community.html">Community & Project Info</a></li> |
| |
| <!-- Roadmap --> |
| <li><a href="/roadmap.html">Roadmap</a></li> |
| |
| <!-- Contribute --> |
| <li><a href="/contributing/how-to-contribute.html">How to Contribute</a></li> |
| |
| |
| <!-- GitHub --> |
| <li> |
| <a href="https://github.com/apache/flink" target="_blank">Flink on GitHub <small><span class="glyphicon glyphicon-new-window"></span></small></a> |
| </li> |
| |
| |
| |
| <!-- Language Switcher --> |
| <li> |
| |
| |
| <a href="/zh/stateful-functions.html">中文版</a> |
| |
| |
| </li> |
| |
| </ul> |
| |
| <ul class="nav navbar-nav navbar-bottom"> |
| <hr /> |
| |
| <!-- Twitter --> |
| <li><a href="https://twitter.com/apacheflink" target="_blank">@ApacheFlink <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| |
| <!-- Visualizer --> |
| <li class=" hidden-md hidden-sm"><a href="/visualizer/" target="_blank">Plan Visualizer <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| |
| <hr /> |
| |
| <li><a href="https://apache.org" target="_blank">Apache Software Foundation <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| |
| <li> |
| <style> |
| .smalllinks:link { |
| display: inline-block !important; background: none; padding-top: 0px; padding-bottom: 0px; padding-right: 0px; min-width: 75px; |
| } |
| </style> |
| |
| <a class="smalllinks" href="https://www.apache.org/licenses/" target="_blank">License</a> <small><span class="glyphicon glyphicon-new-window"></span></small> |
| |
| <a class="smalllinks" href="https://www.apache.org/security/" target="_blank">Security</a> <small><span class="glyphicon glyphicon-new-window"></span></small> |
| |
| <a class="smalllinks" href="https://www.apache.org/foundation/sponsorship.html" target="_blank">Donate</a> <small><span class="glyphicon glyphicon-new-window"></span></small> |
| |
| <a class="smalllinks" href="https://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a> <small><span class="glyphicon glyphicon-new-window"></span></small> |
| </li> |
| |
| </ul> |
| </div><!-- /.navbar-collapse --> |
| </nav> |
| |
| </div> |
| <div class="col-sm-9"> |
| <div class="row-fluid"> |
| |
| <div class="col-sm-12"> |
| <p class="lead"> |
| <strong>Stateful Functions — Event-driven Applications on Apache Flink<sup>®</sup></strong> |
| </p> |
| </div> |
| <div class="col-sm-12"> |
| <hr /> |
| </div> |
| |
| </div> |
| |
| <p>Stateful Functions is an API that <strong>simplifies building distributed stateful applications</strong>. It’s based on functions with persistent state that can interact dynamically with strong consistency guarantees.</p> |
| |
| <div style="line-height:60%;"> |
| <br /> |
| </div> |
| |
| <div class="row front-graphic"> |
| <img src="/img/stateful-functions/statefun-overview.png" width="650px" /> |
| </div> |
| |
| <h3 id="stateful-functions-applications">Stateful Functions Applications</h3> |
| |
| <p>A <em>stateful function</em> is a small piece of logic/code existing in multiple instances that represent entities — similar to <a href="https://www.brianstorti.com/the-actor-model/">actors</a>. Functions are invoked through messages and are:</p> |
| |
| <div class="jumbotron" style="height:auto;padding-top: 18px;padding-bottom: 12px;"> |
| <p style="font-size:100%;"><span class="glyphicon glyphicon glyphicon-check"></span><b> Stateful</b></p> |
| <p style="font-size:100%;">Functions have embedded, fault-tolerant state, accessed locally like a variable.</p> |
| <p style="font-size:100%;"><span class="glyphicon glyphicon glyphicon-check"></span><b> Virtual</b></p> |
| <p style="font-size:100%;">Much like FaaS, functions don't reserve resources — inactive functions don't consume CPU/Memory.</p> |
| </div> |
| |
| <p>Applications are composed of <em>modules</em> of multiple functions that can interact arbitrarily with:</p> |
| |
| <div class="jumbotron" style="height:auto;padding-top: 18px;padding-bottom: 12px;"> |
| <p style="font-size:100%;"><span class="glyphicon glyphicon glyphicon-check"></span><b> Exactly-once Semantics</b></p> |
| <p style="font-size:100%;">State and messaging go hand-in-hand, providing exactly-once message/state semantics.</p> |
| <p style="font-size:100%;"><span class="glyphicon glyphicon glyphicon-check"></span><b> Logical Addressing</b></p> |
| <p style="font-size:100%;">Functions message each other by logical addresses. No service discovery needed.</p> |
| <p style="font-size:100%;"><span class="glyphicon glyphicon glyphicon-check"></span><b> Dynamic and Cyclic Messaging</b></p> |
| <p style="font-size:100%;">Messaging patterns don't need to be pre-defined as dataflows (<i>dynamic</i>) and are also not restricted to DAGs (<i>cyclic</i>).</p> |
| </div> |
| |
| <hr /> |
| |
| <h2 id="a-runtime-built-for-serverless-architectures">A Runtime built for Serverless Architectures</h2> |
| |
| <p>The Stateful Functions runtime is designed to provide a set of properties similar to what characterizes <a href="https://martinfowler.com/articles/serverless.html">serverless functions</a>, but applied to stateful problems.</p> |
| |
| <div style="line-height:60%;"> |
| <br /> |
| </div> |
| |
| <!-- Remote Execution --> |
| <div class="row front-graphic"> |
| <img src="/img/stateful-functions/statefun-remote.png" width="600px" /> |
| </div> |
| |
| <p>The runtime is built on Apache Flink<sup>®</sup>, with the following design principles:</p> |
| |
| <div class="jumbotron" style="height:auto;padding-top: 18px;padding-bottom: 12px;"> |
| <p style="font-size:100%;"><span class="glyphicon glyphicon-edit"></span><b> Logical Compute/State Co-location:</b></p> |
| <p style="font-size:100%;">Messaging, state access/updates and function invocations are managed tightly together. This ensures a high-level of consistency out-of-the-box.</p> |
| <p style="font-size:100%;"><span class="glyphicon glyphicon-edit"></span><b> Physical Compute/State Separation:</b></p> |
| <p style="font-size:100%;">Functions can be executed remotely, with message and state access provided as part of the invocation request. This way, functions can be managed like stateless processes and support rapid scaling, rolling upgrades and other common operational patterns.</p> |
| <p style="font-size:100%;"><span class="glyphicon glyphicon-edit"></span><b> Language Independence:</b></p> |
| <p style="font-size:100%;">Function invocations use a simple HTTP/gRPC-based protocol so that Functions can be easily implemented in various languages.</p> |
| </div> |
| |
| <p>This makes it possible to execute functions on a <strong>Kubernetes deployment</strong>, a <strong>FaaS platform</strong> or <strong>behind a (micro)service</strong>, while providing consistent state and lightweight messaging between functions.</p> |
| |
| <hr /> |
| |
| <h2 id="key-benefits">Key Benefits</h2> |
| |
| <div style="line-height:60%;"> |
| <br /> |
| </div> |
| |
| <!-- Product Marketing Properties --> |
| <div class="row"> |
| <!-- Arbitrary Messaging --> |
| <div class="col-lg-4"> |
| <div class="text-center"> |
| <img class="img-circle" src="/img/stateful-functions/statefun-prop3.png" alt="Arbitrary Messaging" width="90" height="90" /> |
| <h3>Dynamic Messaging</h3> |
| </div> |
| <p align="justify">The API allows you to build and compose functions that communicate dynamic- and arbitrarily with each other. This gives you much more flexibility compared to the acyclic nature of classical stream processing topologies.</p> |
| <p align="justify"><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-stable/concepts/application-building-blocks.html#stateful-functions">Learn More</a></p> |
| </div> |
| <!-- Consistent State --> |
| <div class="col-lg-4"> |
| <div class="text-center"> |
| <img class="img-circle" src="/img/stateful-functions/statefun-prop1.png" alt="Consistent State" width="90" height="90" /> |
| <h3>Consistent State</h3> |
| <p align="justify">Functions can keep local state that is persistent and integrated with the messaging between functions. This gives you the effect of exactly-once state access/updates and guaranteed efficient messaging out-of-the-box.</p> |
| <p align="justify"><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-stable/concepts/application-building-blocks.html#persisted-states">Learn More</a></p> |
| </div> |
| </div> |
| <!-- Multi-language Support --> |
| <div class="col-lg-4"> |
| <div class="text-center"> |
| <img class="img-circle" src="/img/stateful-functions/statefun-prop4.png" alt="Multi-language Support" width="90" height="90" /> |
| <h3>Multi-language Support</h3> |
| </div> |
| <p align="justify">Functions can be implemented in any programming language that can handle HTTP requests or bring up a gRPC server, with initial support for Python. More SDKs will be added for languages like Go, Javascript and Rust. |
| </p> |
| <p align="justify"><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-stable/sdk/modules.html#modules">Learn More</a></p> |
| </div> |
| </div> |
| |
| <hr /> |
| |
| <div class="row"> |
| <!-- No Database Required --> |
| <div class="col-lg-4"> |
| <div class="text-center"> |
| <img class="img-circle" src="/img/stateful-functions/statefun-prop5.png" alt="No Database Required" width="90" height="90" /> |
| <h3>No Database Required</h3> |
| </div> |
| <p align="justify">State durability and fault tolerance build on Apache Flink’s robust distributed snapshots model. This requires nothing but a simple blob storage tier (e.g. S3, GCS, HDFS) to store the state snapshots.</p> |
| <p align="justify"><a href="https://ci.apache.org/projects/flink/flink-docs-stable/internals/stream_checkpointing.html">Learn More</a></p> |
| </div> |
| <!-- Cloud Native --> |
| <div class="col-lg-4"> |
| <div class="text-center"> |
| <img class="img-circle" src="/img/stateful-functions/statefun-prop6.png" alt="Ecosystem Integration" width="90" height="90" /> |
| <h3>Cloud Native</h3> |
| </div> |
| <p align="justify">Stateful Function's approach to state and composition can be combined with the capabilities of modern serverless platforms like Kubernetes, Knative and AWS Lambda.</p> |
| <p align="justify" href="https://thenewstack.io/10-key-attributes-of-cloud-native-applications/"><a href="">Learn More</a></p> |
| </div> |
| <!-- "Stateless" Operation --> |
| <div class="col-lg-4"> |
| <div class="text-center"> |
| <img class="img-circle" src="/img/stateful-functions/statefun-prop2.png" alt="Stateless Operation" width="90" height="90" /> |
| <h3>"Stateless" Operation</h3> |
| </div> |
| <p align="justify">State access is part of the function invocation and so Stateful Functions applications behave like stateless processes that can be managed with the same simplicity and benefits, like rapid scalability, scale-to-zero and rolling/zero-downtime upgrades. |
| </p> |
| <p align="justify"><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-stable/concepts/logical.html#function-lifecycle">Learn More</a></p> |
| </div> |
| </div> |
| |
| <hr /> |
| |
| <h2 id="an-example-transaction-scoring-for-fraud-detection">An Example: Transaction Scoring for Fraud Detection</h2> |
| |
| <div style="line-height:60%;"> |
| <br /> |
| </div> |
| |
| <div class="row"> |
| <div class="col-sm-5"> |
| <img src="/img/stateful-functions/model-score.svg" width="400px" /> |
| </div> |
| <div class="col-sm-7"> |
| <p>Imagine an application that receives financial information and emits alerts for every transaction that exceeds a given threshold fraud score (i.e. fraudulent). To build this example with <b>Stateful Functions</b>, you can define four different functions, each tracking its own state:</p> |
| <p><b>Fraud Count:</b> tracks the total number of reported fraudulent transactions made against an account on a rolling 30 day period.</p> |
| <p><b>Merchant Scorer:</b> returns a trustworthiness score for each merchant, relying on a third party service.</p> |
| <p><b>Transaction Manager:</b> enriches transaction records to create feature vectors for scoring and emits fraud alert events.</p> |
| <p><b>Model:</b> scores transactions based on input feature vectors from the Transaction Manager.</p> |
| </div> |
| </div> |
| |
| <div style="line-height:60%;"> |
| <br /> |
| </div> |
| |
| <p><strong>Keeping track of fraudulent reports</strong></p> |
| |
| <p>The entry points to the application are the “Fraud Confirmation” and “Transactions” <a href="https://ci.apache.org/projects/flink/flink-statefun-docs-stable/concepts/application-building-blocks.html#event-ingress"><em>ingresses</em></a> (e.g. Kafka Topics). As events flow in from “Fraud Confirmation”, the “Fraud Count” function increments its internal counter and sets a 30-day expiration timer on this state. Here, multiple instances of “Fraud Count” will exist — for example, one per customer account. After 30 days, the “Fraud Count” function will receive an expiration message (from itself) and clear its state.</p> |
| |
| <p><strong>Enriching and scoring transactions</strong></p> |
| |
| <p>On receiving events from the “Transactions” ingress, the “Transaction Manager” function messages “Fraud Count” to get the current count of fraud cases reported for the customer account; it also messages the “Merchant Scorer” for the trustworthiness score of the transaction merchant. “Transaction Manager” creates a feature vector with the count of fraud cases reported and the merchant score for the customer account that is then sent to the “Model” function for scoring.</p> |
| |
| <p><strong>Emitting alerts</strong></p> |
| |
| <p>Depending on the score sent back to “Transaction Manager”, it may emit an alert event to the “Alert User” <a href="https://ci.apache.org/projects/flink/flink-statefun-docs-stable/concepts/application-building-blocks.html#event-egress"><em>egress</em></a> if a given threshold is exceeded.</p> |
| |
| <hr /> |
| |
| <h2 id="learn-more">Learn More</h2> |
| |
| <p>If you find these ideas interesting, give Stateful Functions a try and get involved! Check out the <a href="https://ci.apache.org/projects/flink/flink-statefun-docs-stable/getting-started/project-setup.html">Getting Started</a> section for introduction walkthroughs and the <a href="https://ci.apache.org/projects/flink/flink-statefun-docs-stable/">documentation</a> for a deeper look into the internals of Stateful Functions.</p> |
| |
| <div style="line-height:60%;"> |
| <br /> |
| </div> |
| |
| <p><a href="https://github.com/apache/flink-statefun"><img src="/img/stateful-functions/github-logo-link.png" class="rounded-circle" width="20px" height="20px" /></a> <small>GitHub Repository</small></p> |
| |
| <p><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-stable/"><img src="/img/stateful-functions/favicon.png" class="rounded-circle" width="20px" height="20px" /></a> <small>StateFun Documentation</small></p> |
| |
| <p><a href="https://twitter.com/statefun_io"><img src="/img/stateful-functions/twitter-logo-link.png" class="rounded-circle" width="20px" height="20px" /></a> <small>StateFun Twitter</small></p> |
| |
| <!-- Gimmick to make the last link work --> |
| <p><a href="https://twitter.com/statefun_io"><img src="/img/stateful-functions/twitter-logo-link.png" class="rounded-circle" width="0px" height="0px" /></a> <small></small></p> |
| |
| <hr /> |
| |
| <div class="row"> |
| <div class="col-sm-5"> |
| <h3>For a quick overview,</h3> |
| watch <a href="https://youtu.be/fCeHCMJXXM0">this whiteboard session</a>. |
| </div> |
| <div class="col-sm-7"> |
| <div class="bs-example" data-example-id="responsive-embed-16by9-iframe-youtube"> |
| <div class="embed-responsive embed-responsive-16by9"> |
| <iframe class="embed-responsive-item" src="https://www.youtube.com/embed/fCeHCMJXXM0" allowfullscreen=""></iframe>" allowfullscreen> |
| </div> |
| </div> |
| </div> |
| </div> |
| |
| </div> |
| </div> |
| |
| <hr /> |
| |
| <div class="row"> |
| <div class="footer text-center col-sm-12"> |
| <p>Copyright © 2014-2019 <a href="http://apache.org">The Apache Software Foundation</a>. All Rights Reserved.</p> |
| <p>Apache Flink, Flink®, Apache®, the squirrel logo, and the Apache feather logo are either registered trademarks or trademarks of The Apache Software Foundation.</p> |
| <p><a href="/privacy-policy.html">Privacy Policy</a> · <a href="/blog/feed.xml">RSS feed</a></p> |
| </div> |
| </div> |
| </div><!-- /.container --> |
| |
| <!-- Include all compiled plugins (below), or include individual files as needed --> |
| <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script> |
| <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery.matchHeight/0.7.0/jquery.matchHeight-min.js"></script> |
| <script src="/js/codetabs.js"></script> |
| <script src="/js/stickysidebar.js"></script> |
| |
| <!-- Google Analytics --> |
| <script> |
| (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ |
| (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), |
| m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) |
| })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); |
| |
| ga('create', 'UA-52545728-1', 'auto'); |
| ga('send', 'pageview'); |
| </script> |
| </body> |
| </html> |
