Google Git
Sign in
apache / pulsar-site / aee2d9089b6b8e327f04261353077211f4abd636 / . / content / docs / en / 2.2.1 / deploy-bare-metal / index.html
blob: bbcc0e7ce4771b068df61c60bd4526ab1d4a4134 [file] [log] [blame]
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Deploying a cluster on bare metal · Apache Pulsar</title><meta name="viewport" content="width=device-width, initial-scale=1.0"/><meta name="generator" content="Docusaurus"/><meta name="description" content="&gt; ### Tips"/><meta name="docsearch:version" content="2.2.1"/><meta name="docsearch:language" content="en"/><meta property="og:title" content="Deploying a cluster on bare metal · Apache Pulsar"/><meta property="og:type" content="website"/><meta property="og:url" content="https://pulsar.apache.org/"/><meta property="og:description" content="&gt; ### Tips"/><meta name="twitter:card" content="summary"/><meta name="twitter:image" content="https://pulsar.apache.org/img/pulsar.svg"/><link rel="shortcut icon" href="/img/pulsar.ico"/><link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/atom-one-dark.min.css"/><link rel="alternate" type="application/atom+xml" href="https://pulsar.apache.org/blog/atom.xml" title="Apache Pulsar Blog ATOM Feed"/><link rel="alternate" type="application/rss+xml" href="https://pulsar.apache.org/blog/feed.xml" title="Apache Pulsar Blog RSS Feed"/><link rel="stylesheet" href="/css/code-blocks-buttons.css"/><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.0/clipboard.min.js"></script><script type="text/javascript" src="/js/custom.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/en"><img class="logo" src="/img/pulsar.svg" alt="Apache Pulsar"/></a><a href="/en/versions"><h3>2.2.1</h3></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive"><a href="/docs/en/2.2.1/getting-started-standalone" target="_self">Docs</a></li><li class=""><a href="/en/download" target="_self">Download</a></li><li class="siteNavGroupActive"><a href="/docs/en/2.2.1/client-libraries" target="_self">Clients</a></li><li class=""><a href="#restapis" target="_self">REST APIs</a></li><li class=""><a href="#cli" target="_self">Cli</a></li><li class=""><a href="/blog/" target="_self">Blog</a></li><li class=""><a href="#community" target="_self">Community</a></li><li class=""><a href="#apache" target="_self">Apache</a></li><li class=""><a href="https://pulsar-next.staged.apache.org/" target="_self">New Website (Beta)</a></li><span><li><a id="languages-menu" href="#"><img class="languages-icon" src="/img/language.svg" alt="Languages icon"/>English</a><div id="languages-dropdown" class="hide"><ul id="languages-dropdown-items"><li><a href="/docs/ja/2.2.1/deploy-bare-metal">日本語</a></li><li><a href="/docs/fr/2.2.1/deploy-bare-metal">Français</a></li><li><a href="/docs/ko/2.2.1/deploy-bare-metal">한국어</a></li><li><a href="/docs/zh-CN/2.2.1/deploy-bare-metal">中文</a></li><li><a href="/docs/zh-TW/2.2.1/deploy-bare-metal">繁體中文</a></li><li><a href="https://crowdin.com/project/apache-pulsar" target="_blank" rel="noreferrer noopener">Help Translate</a></li></ul></div></li><script>
const languagesMenuItem = document.getElementById("languages-menu");
const languagesDropDown = document.getElementById("languages-dropdown");
languagesMenuItem.addEventListener("click", function(event) {
event.preventDefault();
if (languagesDropDown.className == "hide") {
languagesDropDown.className = "visible";
} else {
languagesDropDown.className = "hide";
}
});
</script></span></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Deployment</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Getting Started</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/pulsar-2.0">Pulsar 2.0</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/getting-started-standalone">Run Pulsar locally</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/getting-started-docker">Pulsar in Docker</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/client-libraries">Client libraries</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Concepts and Architecture</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/concepts-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/concepts-messaging">Messaging</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/concepts-architecture-overview">Architecture</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/concepts-clients">Clients</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/concepts-replication">Geo Replication</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/concepts-multi-tenancy">Multi Tenancy</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/concepts-authentication">Authentication and Authorization</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/concepts-topic-compaction">Topic Compaction</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/concepts-tiered-storage">Tiered Storage</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/concepts-schema-registry">Schema Registry</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Pulsar Functions</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/functions-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/functions-quickstart">Getting started</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/functions-api">API</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/functions-deploying">Deploying functions</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/functions-guarantees">Processing guarantees</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/functions-state">State Storage</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/functions-metrics">Metrics</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Pulsar IO</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/io-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/io-quickstart">Getting started</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/io-managing">Managing Connectors</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/io-connectors">Builtin Connectors</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/io-develop">Developing Connectors</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Pulsar SQL</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/sql-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/sql-getting-started">Getting Started</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/sql-deployment-configurations">Deployment and Configuration</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Deployment</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/deploy-aws">Amazon Web Services</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/deploy-kubernetes">Kubernetes</a></li><li class="navListItem navListItemActive"><a class="navItem" href="/docs/en/2.2.1/deploy-bare-metal">Bare metal</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/deploy-bare-metal-multi-cluster">Bare metal multi-cluster</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/deploy-monitoring">Monitoring</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Administration</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/administration-zk-bk">ZooKeeper and BookKeeper</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/administration-geo">Geo-replication</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/administration-dashboard">Dashboard</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/administration-stats">Pulsar statistics</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/administration-load-distribution">Load distribution</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/administration-proxy">Pulsar proxy</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Security</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/security-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/security-tls-transport">Transport Encryption using TLS</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/security-tls-authentication">Authentication using TLS</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/security-athenz">Authentication using Athenz</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/security-authorization">Authorization and ACLs</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/security-encryption">End-to-End Encryption</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/security-extending">Extending</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Client Libraries</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/client-libraries-java">Java</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/client-libraries-go">Go</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/client-libraries-python">Python</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/client-libraries-cpp">C++</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/client-libraries-websocket">WebSocket</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Admin API</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/admin-api-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/admin-api-clusters">Clusters</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/admin-api-tenants">Tenants</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/admin-api-brokers">Brokers</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/admin-api-namespaces">Namespaces</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/admin-api-permissions">Permissions</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/admin-api-persistent-topics">Persistent topics</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/admin-api-non-persistent-topics">Non-Persistent topics</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/admin-api-partitioned-topics">Partitioned topics</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/admin-api-schemas">Schemas</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Adaptors</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/adaptors-kafka">Kafka client wrapper</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/adaptors-spark">Apache Spark</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/adaptors-storm">Apache Storm</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Cookbooks</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/cookbooks-tiered-storage">Tiered Storage</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/cookbooks-compaction">Topic compaction</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/cookbooks-deduplication">Message deduplication</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/cookbooks-non-persistent">Non-persistent messaging</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/cookbooks-partitioned">Partitioned Topics</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/cookbooks-retention-expiry">Message retention and expiry</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/cookbooks-encryption">Encryption</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/cookbooks-message-queue">Message queue</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Development</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/develop-tools">Simulation tools</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/developing-binary-protocol">Binary protocol</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/develop-schema">Custom schema storage</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/develop-load-manager">Modular load manager</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/develop-cpp">Building Pulsar C++ client</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Reference</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/reference-terminology">Terminology</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/reference-cli-tools">Pulsar CLI tools</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.2.1/reference-configuration">Pulsar configuration</a></li></ul></div></div></section></div><script>
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
var links = coll[i].nextElementSibling.getElementsByTagName('*');
if (checkActiveCategory){
for (var j = 0; j < links.length; j++) {
if (links[j].classList.contains('navListItemActive')){
coll[i].nextElementSibling.classList.toggle('hide');
coll[i].childNodes[1].classList.toggle('rotate');
checkActiveCategory = false;
break;
}
}
}
coll[i].addEventListener('click', function() {
var arrow = this.childNodes[1];
arrow.classList.toggle('rotate');
var content = this.nextElementSibling;
content.classList.toggle('hide');
});
}
document.addEventListener('DOMContentLoaded', function() {
createToggler('#navToggler', '#docsNav', 'docsSliderActive');
createToggler('#tocToggler', 'body', 'tocActive');
var headings = document.querySelector('.toc-headings');
headings && headings.addEventListener('click', function(event) {
var el = event.target;
while(el !== headings){
if (el.tagName === 'A') {
document.body.classList.remove('tocActive');
break;
} else{
el = el.parentNode;
}
}
}, false);
function createToggler(togglerSelector, targetSelector, className) {
var toggler = document.querySelector(togglerSelector);
var target = document.querySelector(targetSelector);
if (!toggler) {
return;
}
toggler.onclick = function(event) {
event.preventDefault();
target.classList.toggle(className);
};
}
});
</script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/pulsar/edit/master/site2/docs/deploy-bare-metal.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Deploying a cluster on bare metal</h1></header><article><div><span><blockquote>
<h3><a class="anchor" aria-hidden="true" id="tips"></a><a href="#tips" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Tips</h3>
<ol>
<li><p>Single-cluster Pulsar installations should be sufficient for all but the most ambitious use cases. If you're interested in experimenting with
Pulsar or using it in a startup or on a single team, we recommend opting for a single cluster. If you do need to run a multi-cluster Pulsar instance,
however, see the guide <a href="/docs/en/2.2.1/deploy-bare-metal-multi-cluster">here</a>.</p></li>
<li><p>If you want to use all builtin <a href="/docs/en/2.2.1/io-overview">Pulsar IO</a> connectors in your Pulsar deployment, you need to download <code>apache-pulsar-io-connectors</code>
package and make sure it is installed under <code>connectors</code> directory in the pulsar directory on every broker node or on every function-worker node if you
have run a separate cluster of function workers for <a href="/docs/en/2.2.1/functions-overview">Pulsar Functions</a>.</p></li>
<li><p>If you want to use <a href="/docs/en/2.2.1/concepts-tiered-storage">Tiered Storage</a> feature in your Pulsar deployment, you need to download <code>apache-pulsar-offloaders</code>
package and make sure it is installed under <code>offloaders</code> directory in the pulsar directory on every broker node. For more details of how to configure
this feature, you could reference this <a href="/docs/en/2.2.1/cookbooks-tiered-storage">Tiered storage cookbook</a>.</p></li>
</ol>
</blockquote>
<p>Deploying a Pulsar cluster involves doing the following (in order):</p>
<ul>
<li>Deploying a <a href="#deploying-a-zookeeper-cluster">ZooKeeper</a> cluster (optional)</li>
<li>Initializing <a href="#initializing-cluster-metadata">cluster metadata</a></li>
<li>Deploying a <a href="#deploying-a-bookkeeper-cluster">BookKeeper</a> cluster</li>
<li>Deploying one or more Pulsar <a href="#deploying-pulsar-brokers">brokers</a></li>
</ul>
<h2><a class="anchor" aria-hidden="true" id="preparation"></a><a href="#preparation" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Preparation</h2>
<h3><a class="anchor" aria-hidden="true" id="requirements"></a><a href="#requirements" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Requirements</h3>
<p>Currently, Pulsar is available for 64-bit <strong>macOS</strong>, <strong>Linux</strong>, and <strong>Windows</strong>. To use Pulsar, you need to install 64-bit JRE/JDK 8 or later versions.</p>
<blockquote>
<p>If you already have an existing ZooKeeper cluster and would like to reuse it, you don't need to prepare the machines
for running ZooKeeper.</p>
</blockquote>
<p>To run Pulsar on bare metal, you are recommended to have:</p>
<ul>
<li>At least 6 Linux machines or VMs
<ul>
<li>3 running <a href="https://zookeeper.apache.org">ZooKeeper</a></li>
<li>3 running a Pulsar broker, and a <a href="https://bookkeeper.apache.org">BookKeeper</a> bookie</li>
</ul></li>
<li>A single <a href="https://en.wikipedia.org/wiki/Domain_Name_System">DNS</a> name covering all of the Pulsar broker hosts</li>
</ul>
<blockquote>
<p>However if you don't have enough machines, or are trying out Pulsar in cluster mode (and expand the cluster later),
you can even deploy Pulsar in one node, where it will run zookeeper, bookie and broker in same machine.</p>
</blockquote>
<p>Each machine in your cluster will need to have <a href="http://www.oracle.com/technetwork/java/javase/downloads/index.html">Java 8</a> or higher installed.</p>
<p>Here's a diagram showing the basic setup:</p>
<p><img src="/docs/assets/pulsar-basic-setup.png" alt="alt-text"></p>
<p>In this diagram, connecting clients need to be able to communicate with the Pulsar cluster using a single URL, in this case <code>pulsar-cluster.acme.com</code>, that abstracts over all of the message-handling brokers. Pulsar message brokers run on machines alongside BookKeeper bookies; brokers and bookies, in turn, rely on ZooKeeper.</p>
<h3><a class="anchor" aria-hidden="true" id="hardware-considerations"></a><a href="#hardware-considerations" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Hardware considerations</h3>
<p>When deploying a Pulsar cluster, we have some basic recommendations that you should keep in mind when capacity planning.</p>
<h4><a class="anchor" aria-hidden="true" id="zookeeper"></a><a href="#zookeeper" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>ZooKeeper</h4>
<p>For machines running ZooKeeper, we recommend using lighter-weight machines or VMs. Pulsar uses ZooKeeper only for periodic coordination- and configuration-related tasks, <em>not</em> for basic operations. If you're running Pulsar on <a href="https://aws.amazon.com/">Amazon Web Services</a> (AWS), for example, a <a href="http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/t2-instances.html">t2.small</a> instance would likely suffice.</p>
<h4><a class="anchor" aria-hidden="true" id="bookies--brokers"></a><a href="#bookies--brokers" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Bookies &amp; Brokers</h4>
<p>For machines running a bookie and a Pulsar broker, we recommend using more powerful machines. For an AWS deployment, for example, <a href="https://aws.amazon.com/blogs/aws/now-available-i3-instances-for-demanding-io-intensive-applications/">i3.4xlarge</a> instances may be appropriate. On those machines we also recommend:</p>
<ul>
<li>Fast CPUs and 10Gbps <a href="https://en.wikipedia.org/wiki/Network_interface_controller">NIC</a> (for Pulsar brokers)</li>
<li>Small and fast <a href="https://en.wikipedia.org/wiki/Solid-state_drive">solid-state drives</a> (SSDs) or <a href="https://en.wikipedia.org/wiki/Hard_disk_drive">hard disk drives</a> (HDDs) with a <a href="https://en.wikipedia.org/wiki/RAID">RAID</a> controller and a battery-backed write cache (for BookKeeper bookies)</li>
</ul>
<h2><a class="anchor" aria-hidden="true" id="installing-the-pulsar-binary-package"></a><a href="#installing-the-pulsar-binary-package" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Installing the Pulsar binary package</h2>
<blockquote>
<p>You'll need to install the Pulsar binary package on <em>each machine in the cluster</em>, including machines running <a href="#deploying-a-zookeeper-cluster">ZooKeeper</a> and <a href="#deploying-a-bookkeeper-cluster">BookKeeper</a>.</p>
</blockquote>
<p>To get started deploying a Pulsar cluster on bare metal, you'll need to download a binary tarball release in one of the following ways:</p>
<ul>
<li>By clicking on the link directly below, which will automatically trigger a download:
<ul>
<li><a href="https://archive.apache.org/dist/pulsar/pulsar-2.2.1/apache-pulsar-2.2.1-bin.tar.gz" download>Pulsar 2.2.1 binary release</a></li>
</ul></li>
<li>From the Pulsar <a href="/download">downloads page</a></li>
<li>From the Pulsar <a href="https://github.com/apache/pulsar/releases/latest">releases page</a> on <a href="https://github.com">GitHub</a></li>
<li>Using <a href="https://www.gnu.org/software/wget">wget</a>:</li>
</ul>
<pre><code class="hljs css language-bash">$ wget https://archive.apache.org/dist/pulsar/pulsar-2.2.1/apache-pulsar-2.2.1-bin.tar.gz
</code></pre>
<p>Once you've downloaded the tarball, untar it and <code>cd</code> into the resulting directory:</p>
<pre><code class="hljs css language-bash">$ tar xvzf apache-pulsar-2.2.1-bin.tar.gz
$ <span class="hljs-built_in">cd</span> apache-pulsar-2.2.1
</code></pre>
<p>The untarred directory contains the following subdirectories:</p>
<table>
<thead>
<tr><th style="text-align:left">Directory</th><th style="text-align:left">Contains</th></tr>
</thead>
<tbody>
<tr><td style="text-align:left"><code>bin</code></td><td style="text-align:left">Pulsar's <a href="/docs/en/2.2.1/reference-cli-tools">command-line tools</a>, such as <a href="/docs/en/2.2.1/reference-cli-tools#pulsar"><code>pulsar</code></a> and <a href="/docs/en/2.2.1/reference-pulsar-admin"><code>pulsar-admin</code></a></td></tr>
<tr><td style="text-align:left"><code>conf</code></td><td style="text-align:left">Configuration files for Pulsar, including for <a href="/docs/en/2.2.1/reference-configuration#broker">broker configuration</a>, <a href="/docs/en/2.2.1/reference-configuration#zookeeper">ZooKeeper configuration</a>, and more</td></tr>
<tr><td style="text-align:left"><code>data</code></td><td style="text-align:left">The data storage directory used by ZooKeeper and BookKeeper.</td></tr>
<tr><td style="text-align:left"><code>lib</code></td><td style="text-align:left">The <a href="https://en.wikipedia.org/wiki/JAR_(file_format)">JAR</a> files used by Pulsar.</td></tr>
<tr><td style="text-align:left"><code>logs</code></td><td style="text-align:left">Logs created by the installation.</td></tr>
</tbody>
</table>
<h2><a class="anchor" aria-hidden="true" id="installing-builtin-connectors-optional"></a><a href="#installing-builtin-connectors-optional" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Installing Builtin Connectors (optional)</h2>
<blockquote>
<p>Since release <code>2.1.0-incubating</code>, Pulsar releases a separate binary distribution, containing all the <code>builtin</code> connectors.
If you would like to enable those <code>builtin</code> connectors, you can follow the instructions as below; otherwise you can
skip this section for now.</p>
</blockquote>
<p>To get started using builtin connectors, you'll need to download the connectors tarball release on every broker node in
one of the following ways:</p>
<ul>
<li><p>by clicking the link below and downloading the release from an Apache mirror:</p>
<ul>
<li><a href="https://archive.apache.org/dist/pulsar/pulsar-2.2.1/apache-pulsar-io-connectors-2.2.1-bin.tar.gz" download>Pulsar IO Connectors 2.2.1 release</a></li>
</ul></li>
<li><p>from the Pulsar <a href="/download">downloads page</a></p></li>
<li><p>from the Pulsar <a href="https://github.com/apache/pulsar/releases/latest">releases page</a></p></li>
<li><p>using <a href="https://www.gnu.org/software/wget">wget</a>:</p>
<pre><code class="hljs css language-shell"><span class="hljs-meta">$</span><span class="bash"> wget https://archive.apache.org/dist/pulsar/pulsar-2.2.1/apache-pulsar-io-connectors-2.2.1-bin.tar.gz</span>
</code></pre></li>
</ul>
<p>Once the tarball is downloaded, in the pulsar directory, untar the io-connectors package and copy the connectors as <code>connectors</code>
in the pulsar directory:</p>
<pre><code class="hljs css language-bash">$ tar xvfz apache-pulsar-io-connectors-2.2.1-bin.tar.gz
// you will find a directory named `apache-pulsar-io-connectors-2.2.1` <span class="hljs-keyword">in</span> the pulsar directory
// <span class="hljs-keyword">then</span> copy the connectors
$ mv apache-pulsar-io-connectors-2.2.1/connectors connectors
$ ls connectors
pulsar-io-aerospike-2.2.1.nar
pulsar-io-cassandra-2.2.1.nar
pulsar-io-kafka-2.2.1.nar
pulsar-io-kinesis-2.2.1.nar
pulsar-io-rabbitmq-2.2.1.nar
pulsar-io-twitter-2.2.1.nar
...
</code></pre>
<h2><a class="anchor" aria-hidden="true" id="installing-tiered-storage-offloaders-optional"></a><a href="#installing-tiered-storage-offloaders-optional" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Installing Tiered Storage Offloaders (optional)</h2>
<blockquote>
<p>Since release <code>2.2.0</code>, Pulsar releases a separate binary distribution, containing the tiered storage offloaders.
If you would like to enable tiered storage feature, you can follow the instructions as below; otherwise you can
skip this section for now.</p>
</blockquote>
<p>To get started using tiered storage offloaders, you'll need to download the offloaders tarball release on every broker node in
one of the following ways:</p>
<ul>
<li><p>by clicking the link below and downloading the release from an Apache mirror:</p>
<ul>
<li><a href="https://archive.apache.org/dist/pulsar/pulsar-2.2.1/apache-pulsar-offloaders-2.2.1-bin.tar.gz" download>Pulsar Tiered Storage Offloaders 2.2.1 release</a></li>
</ul></li>
<li><p>from the Pulsar <a href="/download">downloads page</a></p></li>
<li><p>from the Pulsar <a href="https://github.com/apache/pulsar/releases/latest">releases page</a></p></li>
<li><p>using <a href="https://www.gnu.org/software/wget">wget</a>:</p>
<pre><code class="hljs css language-shell"><span class="hljs-meta">$</span><span class="bash"> wget https://archive.apache.org/dist/pulsar/pulsar-2.2.1/apache-pulsar-offloaders-2.2.1-bin.tar.gz</span>
</code></pre></li>
</ul>
<p>Once the tarball is downloaded, in the pulsar directory, untar the offloaders package and copy the offloaders as <code>offloaders</code>
in the pulsar directory:</p>
<pre><code class="hljs css language-bash">$ tar xvfz apache-pulsar-offloaders-2.2.1-bin.tar.gz
// you will find a directory named `apache-pulsar-offloaders-2.2.1` <span class="hljs-keyword">in</span> the pulsar directory
// <span class="hljs-keyword">then</span> copy the offloaders
$ mv apache-pulsar-offloaders-2.2.1/offloaders offloaders
$ ls offloaders
tiered-storage-jcloud-2.2.1.nar
</code></pre>
<p>For more details of how to configure tiered storage feature, you could reference this <a href="/docs/en/2.2.1/cookbooks-tiered-storage">Tiered storage cookbook</a></p>
<h2><a class="anchor" aria-hidden="true" id="deploying-a-zookeeper-cluster"></a><a href="#deploying-a-zookeeper-cluster" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Deploying a ZooKeeper cluster</h2>
<blockquote>
<p>If you already have an existing zookeeper cluster and would like to use it, you can skip this section.</p>
</blockquote>
<p><a href="https://zookeeper.apache.org">ZooKeeper</a> manages a variety of essential coordination- and configuration-related tasks for Pulsar. To deploy a Pulsar cluster you'll need to deploy ZooKeeper first (before all other components). We recommend deploying a 3-node ZooKeeper cluster. Pulsar does not make heavy use of ZooKeeper, so more lightweight machines or VMs should suffice for running ZooKeeper.</p>
<p>To begin, add all ZooKeeper servers to the configuration specified in <a href="/docs/en/2.2.1/reference-configuration#zookeeper"><code>conf/zookeeper.conf</code></a> (in the Pulsar directory you created <a href="#installing-the-pulsar-binary-package">above</a>). Here's an example:</p>
<pre><code class="hljs css language-properties"><span class="hljs-meta">server.1</span>=<span class="hljs-string">zk1.us-west.example.com:2888:3888</span>
<span class="hljs-meta">server.2</span>=<span class="hljs-string">zk2.us-west.example.com:2888:3888</span>
<span class="hljs-meta">server.3</span>=<span class="hljs-string">zk3.us-west.example.com:2888:3888</span>
</code></pre>
<blockquote>
<p>If you have only one machine to deploy Pulsar, you just need to add one server entry in the configuration file.</p>
</blockquote>
<p>On each host, you need to specify the ID of the node in each node's <code>myid</code> file, which is in each server's <code>data/zookeeper</code> folder by default (this can be changed via the <a href="/docs/en/2.2.1/reference-configuration#zookeeper-dataDir"><code>dataDir</code></a> parameter).</p>
<blockquote>
<p>See the <a href="https://zookeeper.apache.org/doc/r3.4.10/zookeeperAdmin.html#sc_zkMulitServerSetup">Multi-server setup guide</a> in the ZooKeeper documentation for detailed info on <code>myid</code> and more.</p>
</blockquote>
<p>On a ZooKeeper server at <code>zk1.us-west.example.com</code>, for example, you could set the <code>myid</code> value like this:</p>
<pre><code class="hljs css language-bash">$ mkdir -p data/zookeeper
$ <span class="hljs-built_in">echo</span> 1 &gt; data/zookeeper/myid
</code></pre>
<p>On <code>zk2.us-west.example.com</code> the command would be <code>echo 2 &gt; data/zookeeper/myid</code> and so on.</p>
<p>Once each server has been added to the <code>zookeeper.conf</code> configuration and has the appropriate <code>myid</code> entry, you can start ZooKeeper on all hosts (in the background, using nohup) with the <a href="/docs/en/2.2.1/reference-cli-tools#pulsar-daemon"><code>pulsar-daemon</code></a> CLI tool:</p>
<pre><code class="hljs css language-bash">$ bin/pulsar-daemon start zookeeper
</code></pre>
<blockquote>
<p>If you are planning to deploy zookeeper with bookie on the same node, you
need to start zookeeper by using different stats port.</p>
</blockquote>
<p>Start zookeeper with <a href="/docs/en/2.2.1/reference-cli-tools#pulsar-daemon"><code>pulsar-daemon</code></a> CLI tool like:</p>
<pre><code class="hljs css language-bash">$ PULSAR_EXTRA_OPTS=<span class="hljs-string">"-Dstats_server_port=8001"</span> bin/pulsar-daemon start zookeeper
</code></pre>
<h2><a class="anchor" aria-hidden="true" id="initializing-cluster-metadata"></a><a href="#initializing-cluster-metadata" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Initializing cluster metadata</h2>
<p>Once you've deployed ZooKeeper for your cluster, there is some metadata that needs to be written to ZooKeeper for each cluster in your instance. It only needs to be written <strong>once</strong>.</p>
<p>You can initialize this metadata using the <a href="/docs/en/2.2.1/reference-cli-tools#pulsar-initialize-cluster-metadata"><code>initialize-cluster-metadata</code></a> command of the <a href="/docs/en/2.2.1/reference-cli-tools#pulsar"><code>pulsar</code></a> CLI tool. This command can be run on any machine in your ZooKeeper cluster. Here's an example:</p>
<pre><code class="hljs css language-shell"><span class="hljs-meta">$</span><span class="bash"> bin/pulsar initialize-cluster-metadata \</span>
--cluster pulsar-cluster-1 \
--zookeeper zk1.us-west.example.com:2181 \
--configuration-store zk1.us-west.example.com:2181 \
--web-service-url http://pulsar.us-west.example.com:8080 \
--web-service-url-tls https://pulsar.us-west.example.com:8443 \
--broker-service-url pulsar://pulsar.us-west.example.com:6650 \
--broker-service-url-tls pulsar+ssl://pulsar.us-west.example.com:6651
</code></pre>
<p>As you can see from the example above, the following needs to be specified:</p>
<table>
<thead>
<tr><th style="text-align:left">Flag</th><th style="text-align:left">Description</th></tr>
</thead>
<tbody>
<tr><td style="text-align:left"><code>--cluster</code></td><td style="text-align:left">A name for the cluster</td></tr>
<tr><td style="text-align:left"><code>--zookeeper</code></td><td style="text-align:left">A &quot;local&quot; ZooKeeper connection string for the cluster. This connection string only needs to include <em>one</em> machine in the ZooKeeper cluster.</td></tr>
<tr><td style="text-align:left"><code>--configuration-store</code></td><td style="text-align:left">The configuration store connection string for the entire instance. As with the <code>--zookeeper</code> flag, this connection string only needs to include <em>one</em> machine in the ZooKeeper cluster.</td></tr>
<tr><td style="text-align:left"><code>--web-service-url</code></td><td style="text-align:left">The web service URL for the cluster, plus a port. This URL should be a standard DNS name. The default port is 8080 (we don't recommend using a different port).</td></tr>
<tr><td style="text-align:left"><code>--web-service-url-tls</code></td><td style="text-align:left">If you're using <a href="/docs/en/2.2.1/security-tls-transport">TLS</a>, you'll also need to specify a TLS web service URL for the cluster. The default port is 8443 (we don't recommend using a different port).</td></tr>
<tr><td style="text-align:left"><code>--broker-service-url</code></td><td style="text-align:left">A broker service URL enabling interaction with the brokers in the cluster. This URL should use the same DNS name as the web service URL but should use the <code>pulsar</code> scheme instead. The default port is 6650 (we don't recommend using a different port).</td></tr>
<tr><td style="text-align:left"><code>--broker-service-url-tls</code></td><td style="text-align:left">If you're using <a href="/docs/en/2.2.1/security-tls-transport">TLS</a>, you'll also need to specify a TLS web service URL for the cluster as well as a TLS broker service URL for the brokers in the cluster. The default port is 6651 (we don't recommend using a different port).</td></tr>
</tbody>
</table>
<h2><a class="anchor" aria-hidden="true" id="deploying-a-bookkeeper-cluster"></a><a href="#deploying-a-bookkeeper-cluster" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Deploying a BookKeeper cluster</h2>
<p><a href="https://bookkeeper.apache.org">BookKeeper</a> handles all persistent data storage in Pulsar. You will need to deploy a cluster of BookKeeper bookies to use Pulsar. We recommend running a <strong>3-bookie BookKeeper cluster</strong>.</p>
<p>BookKeeper bookies can be configured using the <a href="/docs/en/2.2.1/reference-configuration#bookkeeper"><code>conf/bookkeeper.conf</code></a> configuration file. The most important step in configuring bookies for our purposes here is ensuring that the <a href="/docs/en/2.2.1/reference-configuration#bookkeeper-zkServers"><code>zkServers</code></a> is set to the connection string for the ZooKeeper cluster. Here's an example:</p>
<pre><code class="hljs css language-properties"><span class="hljs-attr">zkServers</span>=<span class="hljs-string">zk1.us-west.example.com:2181,zk2.us-west.example.com:2181,zk3.us-west.example.com:2181</span>
</code></pre>
<p>Once you've appropriately modified the <code>zkServers</code> parameter, you can provide any other configuration modifications you need. You can find a full listing of the available BookKeeper configuration parameters <a href="/docs/en/2.2.1/reference-configuration#bookkeeper">here</a>, although we would recommend consulting the <a href="http://bookkeeper.apache.org/docs/latest/reference/config/">BookKeeper documentation</a> for a more in-depth guide.</p>
<p>Once you've applied the desired configuration in <code>conf/bookkeeper.conf</code>, you can start up a bookie on each of your BookKeeper hosts. You can start up each bookie either in the background, using <a href="https://en.wikipedia.org/wiki/Nohup">nohup</a>, or in the foreground.</p>
<p>To start the bookie in the background, use the <a href="/docs/en/2.2.1/reference-cli-tools#pulsar-daemon"><code>pulsar-daemon</code></a> CLI tool:</p>
<pre><code class="hljs css language-bash">$ bin/pulsar-daemon start bookie
</code></pre>
<p>To start the bookie in the foreground:</p>
<pre><code class="hljs css language-bash">$ bin/bookkeeper bookie
</code></pre>
<p>You can verify that a bookie is working properly by running the <code>bookiesanity</code> command for the <a href="/docs/en/2.2.1/reference-cli-tools#shell">BookKeeper shell</a> on it:</p>
<pre><code class="hljs css language-bash">$ bin/bookkeeper shell bookiesanity
</code></pre>
<p>This will create an ephemeral BookKeeper ledger on the local bookie, write a few entries, read them back, and finally delete the ledger.</p>
<p>After you have started all the bookies, you can use <code>simpletest</code> command for <a href="/docs/en/2.2.1/reference-cli-tools#shell">BookKeeper shell</a> on any bookie node, to
verify all the bookies in the cluster are up running.</p>
<pre><code class="hljs css language-bash">$ bin/bookkeeper shell simpletest --ensemble &lt;num-bookies&gt; --writeQuorum &lt;num-bookies&gt; --ackQuorum &lt;num-bookies&gt; --numEntries &lt;num-entries&gt;
</code></pre>
<p>This command will create a <code>num-bookies</code> sized ledger on the cluster, write a few entries, and finally delete the ledger.</p>
<h2><a class="anchor" aria-hidden="true" id="deploying-pulsar-brokers"></a><a href="#deploying-pulsar-brokers" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Deploying Pulsar brokers</h2>
<p>Pulsar brokers are the last thing you need to deploy in your Pulsar cluster. Brokers handle Pulsar messages and provide Pulsar's administrative interface. We recommend running <strong>3 brokers</strong>, one for each machine that's already running a BookKeeper bookie.</p>
<h3><a class="anchor" aria-hidden="true" id="configuring-brokers"></a><a href="#configuring-brokers" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Configuring Brokers</h3>
<p>The most important element of broker configuration is ensuring that that each broker is aware of the ZooKeeper cluster that you've deployed. Make sure that the <a href="/docs/en/2.2.1/reference-configuration#broker-zookeeperServers"><code>zookeeperServers</code></a> and <a href="/docs/en/2.2.1/reference-configuration#broker-configurationStoreServers"><code>configurationStoreServers</code></a> parameters. In this case, since we only have 1 cluster and no configuration store setup, the <code>configurationStoreServers</code> will point to the same <code>zookeeperServers</code>.</p>
<pre><code class="hljs css language-properties"><span class="hljs-attr">zookeeperServers</span>=<span class="hljs-string">zk1.us-west.example.com:2181,zk2.us-west.example.com:2181,zk3.us-west.example.com:2181</span>
<span class="hljs-attr">configurationStoreServers</span>=<span class="hljs-string">zk1.us-west.example.com:2181,zk2.us-west.example.com:2181,zk3.us-west.example.com:2181</span>
</code></pre>
<p>You also need to specify the cluster name (matching the name that you provided when <a href="#initializing-cluster-metadata">initializing the cluster's metadata</a>:</p>
<pre><code class="hljs css language-properties"><span class="hljs-attr">clusterName</span>=<span class="hljs-string">pulsar-cluster-1</span>
</code></pre>
<blockquote>
<p>If you deploy Pulsar in a one-node cluster, you should update the replication settings in <code>conf/broker.conf</code> to <code>1</code></p>
<pre><code class="hljs css language-properties"><span class="hljs-comment"># Number of bookies to use when creating a ledger</span>
<span class="hljs-attr">managedLedgerDefaultEnsembleSize</span>=<span class="hljs-string">1</span>
<span class="hljs-comment">
# Number of copies to store for each message</span>
<span class="hljs-attr">managedLedgerDefaultWriteQuorum</span>=<span class="hljs-string">1</span>
<span class="hljs-comment">
# Number of guaranteed copies (acks to wait before write is complete)</span>
<span class="hljs-attr">managedLedgerDefaultAckQuorum</span>=<span class="hljs-string">1</span>
</code></pre>
</blockquote>
<h3><a class="anchor" aria-hidden="true" id="enabling-pulsar-functions-optional"></a><a href="#enabling-pulsar-functions-optional" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Enabling Pulsar Functions (optional)</h3>
<p>If you want to enable <a href="/docs/en/2.2.1/functions-overview">Pulsar Functions</a>, you can follow the instructions as below:</p>
<ol>
<li><p>Edit <code>conf/broker.conf</code> to enable function worker, by setting <code>functionsWorkerEnabled</code> to <code>true</code>.</p>
<pre><code class="hljs css language-conf"><span class="hljs-attr">functionsWorkerEnabled</span>=<span class="hljs-literal">true</span>
</code></pre></li>
<li><p>Edit <code>conf/functions_worker.yml</code> and set <code>pulsarFunctionsCluster</code> to the cluster name that you provided when <a href="#initializing-cluster-metadata">initializing the cluster's metadata</a>.</p>
<pre><code class="hljs css language-conf">pulsarFunctionsCluster: pulsar-cluster<span class="hljs-number">-1</span>
</code></pre></li>
</ol>
<h3><a class="anchor" aria-hidden="true" id="starting-brokers"></a><a href="#starting-brokers" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Starting Brokers</h3>
<p>You can then provide any other configuration changes that you'd like in the <a href="/docs/en/2.2.1/reference-configuration#broker"><code>conf/broker.conf</code></a> file. Once you've decided on a configuration, you can start up the brokers for your Pulsar cluster. Like ZooKeeper and BookKeeper, brokers can be started either in the foreground or in the background, using nohup.</p>
<p>You can start a broker in the foreground using the <a href="/docs/en/2.2.1/reference-cli-tools#pulsar-broker"><code>pulsar broker</code></a> command:</p>
<pre><code class="hljs css language-bash">$ bin/pulsar broker
</code></pre>
<p>You can start a broker in the background using the <a href="/docs/en/2.2.1/reference-cli-tools#pulsar-daemon"><code>pulsar-daemon</code></a> CLI tool:</p>
<pre><code class="hljs css language-bash">$ bin/pulsar-daemon start broker
</code></pre>
<p>Once you've successfully started up all the brokers you intend to use, your Pulsar cluster should be ready to go!</p>
<h2><a class="anchor" aria-hidden="true" id="connecting-to-the-running-cluster"></a><a href="#connecting-to-the-running-cluster" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Connecting to the running cluster</h2>
<p>Once your Pulsar cluster is up and running, you should be able to connect with it using Pulsar clients. One such client is the <a href="/docs/en/2.2.1/reference-cli-tools#pulsar-client"><code>pulsar-client</code></a> tool, which is included with the Pulsar binary package. The <code>pulsar-client</code> tool can publish messages to and consume messages from Pulsar topics and thus provides a simple way to make sure that your cluster is running properly.</p>
<p>To use the <code>pulsar-client</code> tool, first modify the client configuration file in <a href="/docs/en/2.2.1/reference-configuration#client"><code>conf/client.conf</code></a> in your binary package. You'll need to change the values for <code>webServiceUrl</code> and <code>brokerServiceUrl</code>, substituting <code>localhost</code> (which is the default), with the DNS name that you've assigned to your broker/bookie hosts. Here's an example:</p>
<pre><code class="hljs css language-properties"><span class="hljs-attr">webServiceUrl</span>=<span class="hljs-string">http://us-west.example.com:8080/</span>
<span class="hljs-attr">brokerServiceurl</span>=<span class="hljs-string">pulsar://us-west.example.com:6650/</span>
</code></pre>
<p>Once you've done that, you can publish a message to Pulsar topic:</p>
<pre><code class="hljs css language-bash">$ bin/pulsar-client produce \
persistent://public/default/<span class="hljs-built_in">test</span> \
-n 1 \
-m <span class="hljs-string">"Hello, Pulsar"</span>
</code></pre>
<blockquote>
<p>You may need to use a different cluster name in the topic if you specified a cluster name different from <code>pulsar-cluster-1</code>.</p>
</blockquote>
<p>This will publish a single message to the Pulsar topic.</p>
<h2><a class="anchor" aria-hidden="true" id="running-functions"></a><a href="#running-functions" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Running Functions</h2>
<blockquote>
<p>If you have <a href="#enabling-pulsar-functions-optional">enabled</a> Pulsar Functions, you can also tryout pulsar functions now.</p>
</blockquote>
<p>Create a ExclamationFunction <code>exclamation</code>.</p>
<pre><code class="hljs css language-bash">bin/pulsar-admin <span class="hljs-built_in">functions</span> create \
--jar examples/api-examples.jar \
--classname org.apache.pulsar.functions.api.examples.ExclamationFunction \
--inputs persistent://public/default/exclamation-input \
--output persistent://public/default/exclamation-output \
--tenant public \
--namespace default \
--name exclamation
</code></pre>
<p>Check if the function is running as expected by <a href="/docs/en/2.2.1/functions-deploying#triggering-pulsar-functions">triggering</a> the function.</p>
<pre><code class="hljs css language-bash">bin/pulsar-admin <span class="hljs-built_in">functions</span> trigger --name exclamation --trigger-value <span class="hljs-string">"hello world"</span>
</code></pre>
<p>You will see output as below:</p>
<pre><code class="hljs css language-shell">hello world!
</code></pre>
</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/en/2.2.1/deploy-kubernetes"><span class="arrow-prev">← </span><span>Kubernetes</span></a><a class="docs-next button" href="/docs/en/2.2.1/deploy-bare-metal-multi-cluster"><span>Bare metal multi-cluster</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#preparation">Preparation</a><ul class="toc-headings"><li><a href="#requirements">Requirements</a></li><li><a href="#hardware-considerations">Hardware considerations</a></li></ul></li><li><a href="#installing-the-pulsar-binary-package">Installing the Pulsar binary package</a></li><li><a href="#installing-builtin-connectors-optional">Installing Builtin Connectors (optional)</a></li><li><a href="#installing-tiered-storage-offloaders-optional">Installing Tiered Storage Offloaders (optional)</a></li><li><a href="#deploying-a-zookeeper-cluster">Deploying a ZooKeeper cluster</a></li><li><a href="#initializing-cluster-metadata">Initializing cluster metadata</a></li><li><a href="#deploying-a-bookkeeper-cluster">Deploying a BookKeeper cluster</a></li><li><a href="#deploying-pulsar-brokers">Deploying Pulsar brokers</a><ul class="toc-headings"><li><a href="#configuring-brokers">Configuring Brokers</a></li><li><a href="#enabling-pulsar-functions-optional">Enabling Pulsar Functions (optional)</a></li><li><a href="#starting-brokers">Starting Brokers</a></li></ul></li><li><a href="#connecting-to-the-running-cluster">Connecting to the running cluster</a></li><li><a href="#running-functions">Running Functions</a></li></ul></nav></div><footer class="nav-footer" id="footer"><section class="copyright">Copyright © 2022 The Apache Software Foundation. All Rights Reserved. Apache, Apache Pulsar and the Apache feather logo are trademarks of The Apache Software Foundation.</section><span><script>
const community = document.querySelector("a[href='#community']").parentNode;
const communityMenu =
'<li>' +
'<a id="community-menu" href="#">Community <span style="font-size: 0.75em">&nbsp;▼</span></a>' +
'<div id="community-dropdown" class="hide">' +
'<ul id="community-dropdown-items">' +
'<li><a href="/en/contact">Contact</a></li>' +
'<li><a href="/en/contributing">Contributing</a></li>' +
'<li><a href="/en/coding-guide">Coding guide</a></li>' +
'<li><a href="/en/events">Events</a></li>' +
'<li><a href="https://twitter.com/Apache_Pulsar" target="_blank">Twitter &#x2750</a></li>' +
'<li><a href="https://github.com/apache/pulsar/wiki" target="_blank">Wiki &#x2750</a></li>' +
'<li><a href="https://github.com/apache/pulsar/issues" target="_blank">Issue tracking &#x2750</a></li>' +
'<li><a href="https://pulsar-summit.org/" target="_blank">Pulsar Summit &#x2750</a></li>' +
'<li>&nbsp;</li>' +
'<li><a href="/en/resources">Resources</a></li>' +
'<li><a href="/en/team">Team</a></li>' +
'<li><a href="/en/powered-by">Powered By</a></li>' +
'</ul>' +
'</div>' +
'</li>';
community.innerHTML = communityMenu;
const communityMenuItem = document.getElementById("community-menu");
const communityDropDown = document.getElementById("community-dropdown");
communityMenuItem.addEventListener("click", function(event) {
event.preventDefault();
if (communityDropDown.className == 'hide') {
communityDropDown.className = 'visible';
} else {
communityDropDown.className = 'hide';
}
});
</script></span></footer></div><script>window.twttr=(function(d,s, id){var js,fjs=d.getElementsByTagName(s)[0],t=window.twttr||{};if(d.getElementById(id))return t;js=d.createElement(s);js.id=id;js.src='https://platform.twitter.com/widgets.js';fjs.parentNode.insertBefore(js, fjs);t._e = [];t.ready = function(f) {t._e.push(f);};return t;}(document, 'script', 'twitter-wjs'));</script></body></html>