Google Git
Sign in
apache / pulsar-site / aee2d9089b6b8e327f04261353077211f4abd636 / . / content / docs / en / 2.6.4 / functions-overview.html
blob: 8902fb7c3b0a6bfde0919d1b0c445ebfd5d1f9da [file] [log] [blame]
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Pulsar Functions overview · Apache Pulsar</title><meta name="viewport" content="width=device-width, initial-scale=1.0"/><meta name="generator" content="Docusaurus"/><meta name="description" content="**Pulsar Functions** are lightweight compute processes that"/><meta name="docsearch:version" content="2.6.4"/><meta name="docsearch:language" content="en"/><meta property="og:title" content="Pulsar Functions overview · Apache Pulsar"/><meta property="og:type" content="website"/><meta property="og:url" content="https://pulsar.apache.org/"/><meta property="og:description" content="**Pulsar Functions** are lightweight compute processes that"/><meta name="twitter:card" content="summary"/><meta name="twitter:image" content="https://pulsar.apache.org/img/pulsar.svg"/><link rel="shortcut icon" href="/img/pulsar.ico"/><link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/atom-one-dark.min.css"/><link rel="alternate" type="application/atom+xml" href="https://pulsar.apache.org/blog/atom.xml" title="Apache Pulsar Blog ATOM Feed"/><link rel="alternate" type="application/rss+xml" href="https://pulsar.apache.org/blog/feed.xml" title="Apache Pulsar Blog RSS Feed"/><link rel="stylesheet" href="/css/code-blocks-buttons.css"/><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.0/clipboard.min.js"></script><script type="text/javascript" src="/js/custom.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/en"><img class="logo" src="/img/pulsar.svg" alt="Apache Pulsar"/></a><a href="/en/versions"><h3>2.6.4</h3></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive"><a href="/docs/en/2.6.4/getting-started-standalone" target="_self">Docs</a></li><li class=""><a href="/en/download" target="_self">Download</a></li><li class="siteNavGroupActive"><a href="/docs/en/2.6.4/client-libraries" target="_self">Clients</a></li><li class=""><a href="#restapis" target="_self">REST APIs</a></li><li class=""><a href="#cli" target="_self">Cli</a></li><li class=""><a href="/blog/" target="_self">Blog</a></li><li class=""><a href="#community" target="_self">Community</a></li><li class=""><a href="#apache" target="_self">Apache</a></li><li class=""><a href="https://pulsar-next.staged.apache.org/" target="_self">New Website (Beta)</a></li><span><li><a id="languages-menu" href="#"><img class="languages-icon" src="/img/language.svg" alt="Languages icon"/>English</a><div id="languages-dropdown" class="hide"><ul id="languages-dropdown-items"><li><a href="/docs/ja/2.6.4/functions-overview">日本語</a></li><li><a href="/docs/fr/2.6.4/functions-overview">Français</a></li><li><a href="/docs/ko/2.6.4/functions-overview">한국어</a></li><li><a href="/docs/zh-CN/2.6.4/functions-overview">中文</a></li><li><a href="/docs/zh-TW/2.6.4/functions-overview">繁體中文</a></li><li><a href="https://crowdin.com/project/apache-pulsar" target="_blank" rel="noreferrer noopener">Help Translate</a></li></ul></div></li><script>
const languagesMenuItem = document.getElementById("languages-menu");
const languagesDropDown = document.getElementById("languages-dropdown");
languagesMenuItem.addEventListener("click", function(event) {
event.preventDefault();
if (languagesDropDown.className == "hide") {
languagesDropDown.className = "visible";
} else {
languagesDropDown.className = "hide";
}
});
</script></span></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Pulsar Functions</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Get Started</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/pulsar-2.0">Pulsar 2.0</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/getting-started-standalone">Run Pulsar locally</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/getting-started-docker">Run Pulsar in Docker</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/getting-started-helm">Run Pulsar in Kubernetes</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/client-libraries">Use Pulsar with client libraries</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Concepts and Architecture</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/concepts-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/concepts-messaging">Messaging</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/concepts-architecture-overview">Architecture</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/concepts-clients">Clients</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/concepts-replication">Geo Replication</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/concepts-multi-tenancy">Multi Tenancy</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/concepts-authentication">Authentication and Authorization</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/concepts-topic-compaction">Topic Compaction</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/concepts-tiered-storage">Tiered Storage</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/concepts-proxy-sni-routing">Proxy support with SNI routing</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Pulsar Schema</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/schema-get-started">Get started</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/schema-understand">Understand schema</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/schema-evolution-compatibility">Schema evolution and compatibility</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/schema-manage">Manage schema</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Pulsar Functions</h3><ul class=""><li class="navListItem navListItemActive"><a class="navItem" href="/docs/en/2.6.4/functions-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/functions-worker">Setup: Pulsar Functions Worker</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/functions-runtime">Setup: Configure Functions runtime</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/functions-develop">How-to: Develop</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/functions-debug">How-to: Debug</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/functions-deploy">How-to: Deploy</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/functions-cli">Reference: CLI</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/window-functions-context">Window Functions: Context</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Pulsar IO</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/io-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/io-quickstart">Get started</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/io-use">Use</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/io-debug">Debug</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/io-connectors">Built-in connector</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/io-cdc">CDC connector</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/io-develop">Develop</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/io-cli">CLI</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Pulsar SQL</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/sql-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/sql-getting-started">Query data</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/sql-deployment-configurations">Configuration and deployment</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/sql-rest-api">REST APIs</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Kubernetes (Helm)</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/helm-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/helm-prepare">Prepare</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/helm-install">Install</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/helm-deploy">Deployment</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/helm-upgrade">Upgrade</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/helm-tools">Required Tools</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Deployment</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/deploy-aws">Amazon Web Services</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/deploy-kubernetes">Kubernetes</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/deploy-bare-metal">Bare metal</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/deploy-bare-metal-multi-cluster">Bare metal multi-cluster</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/deploy-monitoring">Monitoring</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Administration</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/administration-zk-bk">ZooKeeper and BookKeeper</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/administration-geo">Geo-replication</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/administration-pulsar-manager">Pulsar Manager</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/administration-stats">Pulsar statistics</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/administration-load-balance">Load balance</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/administration-proxy">Pulsar proxy</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/administration-upgrade">Upgrade</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Security</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/security-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/security-tls-transport">Transport Encryption using TLS</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/security-tls-authentication">Authentication using TLS</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/security-tls-keystore">Using TLS with KeyStore configure</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/security-jwt">Authentication using JWT</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/security-athenz">Authentication using Athenz</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/security-kerberos">Authentication using Kerberos</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/security-oauth2">Authentication using OAuth 2.0 access tokens</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/security-authorization">Authorization and ACLs</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/security-encryption">End-to-End Encryption</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/security-extending">Extending</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/security-bouncy-castle">Bouncy Castle Providers</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Performance</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/performance-pulsar-perf">Pulsar Perf</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Client Libraries</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/client-libraries-java">Java</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/client-libraries-go">Go</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/client-libraries-python">Python</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/client-libraries-cpp">C++</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/client-libraries-node">Node.js</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/client-libraries-websocket">WebSocket</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/client-libraries-dotnet">C#</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Admin API</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/admin-api-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/admin-api-clusters">Clusters</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/admin-api-tenants">Tenants</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/admin-api-brokers">Brokers</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/admin-api-namespaces">Namespaces</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/admin-api-permissions">Permissions</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/admin-api-persistent-topics">Persistent topics</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/admin-api-non-persistent-topics">Non-Persistent topics</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/admin-api-partitioned-topics">Partitioned topics</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/admin-api-non-partitioned-topics">Non-Partitioned topics</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/admin-api-schemas">Schemas</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/admin-api-functions">Functions</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Adaptors</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/adaptors-kafka">Kafka client wrapper</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/adaptors-spark">Apache Spark</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/adaptors-storm">Apache Storm</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Cookbooks</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/cookbooks-tiered-storage">Tiered Storage</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/cookbooks-compaction">Topic compaction</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/cookbooks-deduplication">Message deduplication</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/cookbooks-non-persistent">Non-persistent messaging</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/cookbooks-partitioned">Partitioned Topics</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/cookbooks-retention-expiry">Message retention and expiry</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/cookbooks-encryption">Encryption</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/cookbooks-message-queue">Message queue</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/cookbooks-bookkeepermetadata">BookKeeper Ledger Metadata</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Development</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/develop-tools">Simulation tools</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/developing-binary-protocol">Binary protocol</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/develop-schema">Custom schema storage</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/develop-load-manager">Modular load manager</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/develop-cpp">Building Pulsar C++ client</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Reference</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/reference-terminology">Terminology</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/reference-cli-tools">Pulsar CLI tools</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/reference-configuration">Pulsar configuration</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.6.4/reference-metrics">Pulsar Metrics</a></li></ul></div></div></section></div><script>
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
var links = coll[i].nextElementSibling.getElementsByTagName('*');
if (checkActiveCategory){
for (var j = 0; j < links.length; j++) {
if (links[j].classList.contains('navListItemActive')){
coll[i].nextElementSibling.classList.toggle('hide');
coll[i].childNodes[1].classList.toggle('rotate');
checkActiveCategory = false;
break;
}
}
}
coll[i].addEventListener('click', function() {
var arrow = this.childNodes[1];
arrow.classList.toggle('rotate');
var content = this.nextElementSibling;
content.classList.toggle('hide');
});
}
document.addEventListener('DOMContentLoaded', function() {
createToggler('#navToggler', '#docsNav', 'docsSliderActive');
createToggler('#tocToggler', 'body', 'tocActive');
var headings = document.querySelector('.toc-headings');
headings && headings.addEventListener('click', function(event) {
var el = event.target;
while(el !== headings){
if (el.tagName === 'A') {
document.body.classList.remove('tocActive');
break;
} else{
el = el.parentNode;
}
}
}, false);
function createToggler(togglerSelector, targetSelector, className) {
var toggler = document.querySelector(togglerSelector);
var target = document.querySelector(targetSelector);
if (!toggler) {
return;
}
toggler.onclick = function(event) {
event.preventDefault();
target.classList.toggle(className);
};
}
});
</script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/pulsar/edit/master/site2/docs/functions-overview.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Pulsar Functions overview</h1></header><article><div><span><p><strong>Pulsar Functions</strong> are lightweight compute processes that</p>
<ul>
<li>consume messages from one or more Pulsar topics,</li>
<li>apply a user-supplied processing logic to each message,</li>
<li>publish the results of the computation to another topic.</li>
</ul>
<h2><a class="anchor" aria-hidden="true" id="goals"></a><a href="#goals" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Goals</h2>
<p>With Pulsar Functions, you can create complex processing logic without deploying a separate neighboring system (such as <a href="http://storm.apache.org/">Apache Storm</a>, <a href="https://heron.incubator.apache.org/">Apache Heron</a>, <a href="https://flink.apache.org/">Apache Flink</a>). Pulsar Functions are computing infrastructure of Pulsar messaging system. The core goal is tied to a series of other goals:</p>
<ul>
<li>Developer productivity (language-native vs Pulsar Functions SDK functions)</li>
<li>Easy troubleshooting</li>
<li>Operational simplicity (no need for an external processing system)</li>
</ul>
<h2><a class="anchor" aria-hidden="true" id="inspirations"></a><a href="#inspirations" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Inspirations</h2>
<p>Pulsar Functions are inspired by (and take cues from) several systems and paradigms:</p>
<ul>
<li>Stream processing engines such as <a href="http://storm.apache.org/">Apache Storm</a>, <a href="https://apache.github.io/incubator-heron">Apache Heron</a>, and <a href="https://flink.apache.org">Apache Flink</a></li>
<li>&quot;Serverless&quot; and &quot;Function as a Service&quot; (FaaS) cloud platforms like <a href="https://aws.amazon.com/lambda/">Amazon Web Services Lambda</a>, <a href="https://cloud.google.com/functions/">Google Cloud Functions</a>, and <a href="https://azure.microsoft.com/en-us/services/functions/">Azure Cloud Functions</a></li>
</ul>
<p>Pulsar Functions can be described as</p>
<ul>
<li><a href="https://aws.amazon.com/lambda/">Lambda</a>-style functions that are</li>
<li>specifically designed to use Pulsar as a message bus.</li>
</ul>
<h2><a class="anchor" aria-hidden="true" id="programming-model"></a><a href="#programming-model" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Programming model</h2>
<p>Pulsar Functions provide a wide range of functionality, and the core programming model is simple. Functions receive messages from one or more <strong>input <a href="/docs/en/2.6.4/reference-terminology#topic">topics</a></strong>. Each time a message is received, the function will complete the following tasks.</p>
<ul>
<li>Apply some processing logic to the input and write output to:
<ul>
<li>An <strong>output topic</strong> in Pulsar</li>
<li><a href="/docs/en/2.6.4/functions-develop#state-storage">Apache BookKeeper</a></li>
</ul></li>
<li>Write logs to a <strong>log topic</strong> (potentially for debugging purposes)</li>
<li>Increment a <a href="#word-count-example">counter</a></li>
</ul>
<p><img src="/docs/assets/pulsar-functions-overview.png" alt="Pulsar Functions core programming model"></p>
<p>You can use Pulsar Functions to set up the following processing chain:</p>
<ul>
<li>A Python function listens for the <code>raw-sentences</code> topic and &quot;sanitizes&quot; incoming strings (removing extraneous whitespace and converting all characters to lowercase) and then publishes the results to a <code>sanitized-sentences</code> topic.</li>
<li>A Java function listens for the <code>sanitized-sentences</code> topic, counts the number of times each word appears within a specified time window, and publishes the results to a <code>results</code> topic</li>
<li>Finally, a Python function listens for the <code>results</code> topic and writes the results to a MySQL table.</li>
</ul>
<h3><a class="anchor" aria-hidden="true" id="word-count-example"></a><a href="#word-count-example" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Word count example</h3>
<p>If you implement the classic word count example using Pulsar Functions, it looks something like this:</p>
<p><img src="/docs/assets/pulsar-functions-word-count.png" alt="Pulsar Functions word count example"></p>
<p>To write the function in Java with <a href="/docs/en/2.6.4/functions-develop#available-apis">Pulsar Functions SDK for Java</a>, you can write the function as follows.</p>
<pre><code class="hljs css language-java"><span class="hljs-keyword">package</span> org.example.functions;
<span class="hljs-keyword">import</span> org.apache.pulsar.functions.api.Context;
<span class="hljs-keyword">import</span> org.apache.pulsar.functions.api.Function;
<span class="hljs-keyword">import</span> java.util.Arrays;
<span class="hljs-keyword">public</span> <span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">WordCountFunction</span> <span class="hljs-keyword">implements</span> <span class="hljs-title">Function</span>&lt;<span class="hljs-title">String</span>, <span class="hljs-title">Void</span>&gt; </span>{
<span class="hljs-comment">// This function is invoked every time a message is published to the input topic</span>
<span class="hljs-meta">@Override</span>
<span class="hljs-function"><span class="hljs-keyword">public</span> Void <span class="hljs-title">process</span><span class="hljs-params">(String input, Context context)</span> <span class="hljs-keyword">throws</span> Exception </span>{
Arrays.asList(input.split(<span class="hljs-string">" "</span>)).forEach(word -&gt; {
String counterKey = word.toLowerCase();
context.incrCounter(counterKey, <span class="hljs-number">1</span>);
});
<span class="hljs-keyword">return</span> <span class="hljs-keyword">null</span>;
}
}
</code></pre>
<p>Bundle and build the JAR file to be deployed, and then deploy it in your Pulsar cluster using the <a href="/docs/en/2.6.4/functions-deploy#command-line-interface">command line</a> as follows.</p>
<pre><code class="hljs css language-bash">$ bin/pulsar-admin <span class="hljs-built_in">functions</span> create \
--jar target/my-jar-with-dependencies.jar \
--classname org.example.functions.WordCountFunction \
--tenant public \
--namespace default \
--name word-count \
--inputs persistent://public/default/sentences \
--output persistent://public/default/count
</code></pre>
<h3><a class="anchor" aria-hidden="true" id="content-based-routing-example"></a><a href="#content-based-routing-example" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Content-based routing example</h3>
<p>Pulsar Functions are used in many cases. The following is a sophisticated example that involves content-based routing.</p>
<p>For example, a function takes items (strings) as input and publishes them to either a <code>fruits</code> or <code>vegetables</code> topic, depending on the item. Or, if an item is neither fruit nor vegetable, a warning is logged to a <a href="/docs/en/2.6.4/functions-develop#logger">log topic</a>. The following is a visual representation.</p>
<p><img src="/docs/assets/pulsar-functions-routing-example.png" alt="Pulsar Functions routing example"></p>
<p>If you implement this routing functionality in Python, it looks something like this:</p>
<pre><code class="hljs css language-python"><span class="hljs-keyword">from</span> pulsar <span class="hljs-keyword">import</span> Function
<span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">RoutingFunction</span><span class="hljs-params">(Function)</span>:</span>
<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">__init__</span><span class="hljs-params">(self)</span>:</span>
self.fruits_topic = <span class="hljs-string">"persistent://public/default/fruits"</span>
self.vegetables_topic = <span class="hljs-string">"persistent://public/default/vegetables"</span>
<span class="hljs-meta"> @staticmethod</span>
<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">is_fruit</span><span class="hljs-params">(item)</span>:</span>
<span class="hljs-keyword">return</span> item <span class="hljs-keyword">in</span> [<span class="hljs-string">b"apple"</span>, <span class="hljs-string">b"orange"</span>, <span class="hljs-string">b"pear"</span>, <span class="hljs-string">b"other fruits..."</span>]
<span class="hljs-meta"> @staticmethod</span>
<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">is_vegetable</span><span class="hljs-params">(item)</span>:</span>
<span class="hljs-keyword">return</span> item <span class="hljs-keyword">in</span> [<span class="hljs-string">b"carrot"</span>, <span class="hljs-string">b"lettuce"</span>, <span class="hljs-string">b"radish"</span>, <span class="hljs-string">b"other vegetables..."</span>]
<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">process</span><span class="hljs-params">(self, item, context)</span>:</span>
<span class="hljs-keyword">if</span> self.is_fruit(item):
context.publish(self.fruits_topic, item)
<span class="hljs-keyword">elif</span> self.is_vegetable(item):
context.publish(self.vegetables_topic, item)
<span class="hljs-keyword">else</span>:
warning = <span class="hljs-string">"The item {0} is neither a fruit nor a vegetable"</span>.format(item)
context.get_logger().warn(warning)
</code></pre>
<p>If this code is stored in <code>~/router.py</code>, then you can deploy it in your Pulsar cluster using the <a href="/docs/en/2.6.4/functions-deploy#command-line-interface">command line</a> as follows.</p>
<pre><code class="hljs css language-bash">$ bin/pulsar-admin <span class="hljs-built_in">functions</span> create \
--py ~/router.py \
--classname router.RoutingFunction \
--tenant public \
--namespace default \
--name route-fruit-veg \
--inputs persistent://public/default/basket-items
</code></pre>
<h3><a class="anchor" aria-hidden="true" id="functions-messages-and-message-types"></a><a href="#functions-messages-and-message-types" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Functions, messages and message types</h3>
<p>Pulsar Functions take byte arrays as inputs and spit out byte arrays as output. However in languages that support typed interfaces(Java), you can write typed Functions, and bind messages to types in the following ways.</p>
<ul>
<li><a href="/docs/en/2.6.4/functions-develop#schema-registry">Schema Registry</a></li>
<li><a href="/docs/en/2.6.4/functions-develop#serde">SerDe</a></li>
</ul>
<h2><a class="anchor" aria-hidden="true" id="fully-qualified-function-name-fqfn"></a><a href="#fully-qualified-function-name-fqfn" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Fully Qualified Function Name (FQFN)</h2>
<p>Each Pulsar Function has a <strong>Fully Qualified Function Name</strong> (FQFN) that consists of three elements: the function tenant, namespace, and function name. FQFN looks like this:</p>
<pre><code class="hljs css language-http">tenant/namespace/name
</code></pre>
<p>FQFNs enable you to create multiple functions with the same name provided that they are in different namespaces.</p>
<h2><a class="anchor" aria-hidden="true" id="supported-languages"></a><a href="#supported-languages" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Supported languages</h2>
<p>Currently, you can write Pulsar Functions in Java, Python, and Go. For details, refer to <a href="/docs/en/2.6.4/functions-develop">Develop Pulsar Functions</a>.</p>
<h2><a class="anchor" aria-hidden="true" id="processing-guarantees"></a><a href="#processing-guarantees" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Processing guarantees</h2>
<p>Pulsar Functions provide three different messaging semantics that you can apply to any function.</p>
<table>
<thead>
<tr><th style="text-align:left">Delivery semantics</th><th style="text-align:left">Description</th></tr>
</thead>
<tbody>
<tr><td style="text-align:left"><strong>At-most-once</strong> delivery</td><td style="text-align:left">Each message sent to the function is likely to be processed, or not to be processed (hence &quot;at most&quot;).</td></tr>
<tr><td style="text-align:left"><strong>At-least-once</strong> delivery</td><td style="text-align:left">Each message sent to the function can be processed more than once (hence the &quot;at least&quot;).</td></tr>
<tr><td style="text-align:left"><strong>Effectively-once</strong> delivery</td><td style="text-align:left">Each message sent to the function will have one output associated with it.</td></tr>
</tbody>
</table>
<h3><a class="anchor" aria-hidden="true" id="apply-processing-guarantees-to-a-function"></a><a href="#apply-processing-guarantees-to-a-function" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Apply processing guarantees to a function</h3>
<p>You can set the processing guarantees for a Pulsar Function when you create the Function. The following <a href="/docs/en/2.6.4/reference-pulsar-admin#create-1"><code>pulsar-function create</code></a> command creates a function with effectively-once guarantees applied.</p>
<pre><code class="hljs css language-bash">$ bin/pulsar-admin <span class="hljs-built_in">functions</span> create \
--name my-effectively-once-function \
--processing-guarantees EFFECTIVELY_ONCE \
<span class="hljs-comment"># Other function configs</span>
</code></pre>
<p>The available options for <code>--processing-guarantees</code> are:</p>
<ul>
<li><code>ATMOST_ONCE</code></li>
<li><code>ATLEAST_ONCE</code></li>
<li><code>EFFECTIVELY_ONCE</code></li>
</ul>
<blockquote>
<p>By default, Pulsar Functions provide at-least-once delivery guarantees. So if you create a function without supplying a value for the <code>--processingGuarantees</code> flag, the function provides at-least-once guarantees.</p>
</blockquote>
<h3><a class="anchor" aria-hidden="true" id="update-the-processing-guarantees-of-a-function"></a><a href="#update-the-processing-guarantees-of-a-function" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Update the processing guarantees of a function</h3>
<p>You can change the processing guarantees applied to a function using the <a href="/docs/en/2.6.4/reference-pulsar-admin#update-1"><code>update</code></a> command. The following is an example.</p>
<pre><code class="hljs css language-bash">$ bin/pulsar-admin <span class="hljs-built_in">functions</span> update \
--processing-guarantees ATMOST_ONCE \
<span class="hljs-comment"># Other function configs</span>
</code></pre>
</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/en/2.6.4/schema-manage"><span class="arrow-prev">← </span><span>Manage schema</span></a><a class="docs-next button" href="/docs/en/2.6.4/functions-worker"><span>Setup: Pulsar Functions Worker</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#goals">Goals</a></li><li><a href="#inspirations">Inspirations</a></li><li><a href="#programming-model">Programming model</a><ul class="toc-headings"><li><a href="#word-count-example">Word count example</a></li><li><a href="#content-based-routing-example">Content-based routing example</a></li><li><a href="#functions-messages-and-message-types">Functions, messages and message types</a></li></ul></li><li><a href="#fully-qualified-function-name-fqfn">Fully Qualified Function Name (FQFN)</a></li><li><a href="#supported-languages">Supported languages</a></li><li><a href="#processing-guarantees">Processing guarantees</a><ul class="toc-headings"><li><a href="#apply-processing-guarantees-to-a-function">Apply processing guarantees to a function</a></li><li><a href="#update-the-processing-guarantees-of-a-function">Update the processing guarantees of a function</a></li></ul></li></ul></nav></div><footer class="nav-footer" id="footer"><section class="copyright">Copyright © 2022 The Apache Software Foundation. All Rights Reserved. Apache, Apache Pulsar and the Apache feather logo are trademarks of The Apache Software Foundation.</section><span><script>
const community = document.querySelector("a[href='#community']").parentNode;
const communityMenu =
'<li>' +
'<a id="community-menu" href="#">Community <span style="font-size: 0.75em">&nbsp;▼</span></a>' +
'<div id="community-dropdown" class="hide">' +
'<ul id="community-dropdown-items">' +
'<li><a href="/en/contact">Contact</a></li>' +
'<li><a href="/en/contributing">Contributing</a></li>' +
'<li><a href="/en/coding-guide">Coding guide</a></li>' +
'<li><a href="/en/events">Events</a></li>' +
'<li><a href="https://twitter.com/Apache_Pulsar" target="_blank">Twitter &#x2750</a></li>' +
'<li><a href="https://github.com/apache/pulsar/wiki" target="_blank">Wiki &#x2750</a></li>' +
'<li><a href="https://github.com/apache/pulsar/issues" target="_blank">Issue tracking &#x2750</a></li>' +
'<li><a href="https://pulsar-summit.org/" target="_blank">Pulsar Summit &#x2750</a></li>' +
'<li>&nbsp;</li>' +
'<li><a href="/en/resources">Resources</a></li>' +
'<li><a href="/en/team">Team</a></li>' +
'<li><a href="/en/powered-by">Powered By</a></li>' +
'</ul>' +
'</div>' +
'</li>';
community.innerHTML = communityMenu;
const communityMenuItem = document.getElementById("community-menu");
const communityDropDown = document.getElementById("community-dropdown");
communityMenuItem.addEventListener("click", function(event) {
event.preventDefault();
if (communityDropDown.className == 'hide') {
communityDropDown.className = 'visible';
} else {
communityDropDown.className = 'hide';
}
});
</script></span></footer></div><script>window.twttr=(function(d,s, id){var js,fjs=d.getElementsByTagName(s)[0],t=window.twttr||{};if(d.getElementById(id))return t;js=d.createElement(s);js.id=id;js.src='https://platform.twitter.com/widgets.js';fjs.parentNode.insertBefore(js, fjs);t._e = [];t.ready = function(f) {t._e.push(f);};return t;}(document, 'script', 'twitter-wjs'));</script></body></html>