Google Git
Sign in
apache / pulsar-site / aee2d9089b6b8e327f04261353077211f4abd636 / . / content / docs / en / 2.7.3 / sql-deployment-configurations / index.html
blob: ccf31f00bbfba113f226273bd0de4f8478235622 [file] [log] [blame]
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Pulsar SQL configuration and deployment · Apache Pulsar</title><meta name="viewport" content="width=device-width, initial-scale=1.0"/><meta name="generator" content="Docusaurus"/><meta name="description" content="You can configure Presto Pulsar connector and deploy a cluster with the following instruction."/><meta name="docsearch:version" content="2.7.3"/><meta name="docsearch:language" content="en"/><meta property="og:title" content="Pulsar SQL configuration and deployment · Apache Pulsar"/><meta property="og:type" content="website"/><meta property="og:url" content="https://pulsar.apache.org/"/><meta property="og:description" content="You can configure Presto Pulsar connector and deploy a cluster with the following instruction."/><meta name="twitter:card" content="summary"/><meta name="twitter:image" content="https://pulsar.apache.org/img/pulsar.svg"/><link rel="shortcut icon" href="/img/pulsar.ico"/><link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/atom-one-dark.min.css"/><link rel="alternate" type="application/atom+xml" href="https://pulsar.apache.org/blog/atom.xml" title="Apache Pulsar Blog ATOM Feed"/><link rel="alternate" type="application/rss+xml" href="https://pulsar.apache.org/blog/feed.xml" title="Apache Pulsar Blog RSS Feed"/><link rel="stylesheet" href="/css/code-blocks-buttons.css"/><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.0/clipboard.min.js"></script><script type="text/javascript" src="/js/custom.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/en"><img class="logo" src="/img/pulsar.svg" alt="Apache Pulsar"/></a><a href="/en/versions"><h3>2.7.3</h3></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive"><a href="/docs/en/2.7.3/getting-started-standalone" target="_self">Docs</a></li><li class=""><a href="/en/download" target="_self">Download</a></li><li class="siteNavGroupActive"><a href="/docs/en/2.7.3/client-libraries" target="_self">Clients</a></li><li class=""><a href="#restapis" target="_self">REST APIs</a></li><li class=""><a href="#cli" target="_self">Cli</a></li><li class=""><a href="/blog/" target="_self">Blog</a></li><li class=""><a href="#community" target="_self">Community</a></li><li class=""><a href="#apache" target="_self">Apache</a></li><li class=""><a href="https://pulsar-next.staged.apache.org/" target="_self">New Website (Beta)</a></li><span><li><a id="languages-menu" href="#"><img class="languages-icon" src="/img/language.svg" alt="Languages icon"/>English</a><div id="languages-dropdown" class="hide"><ul id="languages-dropdown-items"><li><a href="/docs/ja/2.7.3/sql-deployment-configurations">日本語</a></li><li><a href="/docs/fr/2.7.3/sql-deployment-configurations">Français</a></li><li><a href="/docs/ko/2.7.3/sql-deployment-configurations">한국어</a></li><li><a href="/docs/zh-CN/2.7.3/sql-deployment-configurations">中文</a></li><li><a href="/docs/zh-TW/2.7.3/sql-deployment-configurations">繁體中文</a></li><li><a href="https://crowdin.com/project/apache-pulsar" target="_blank" rel="noreferrer noopener">Help Translate</a></li></ul></div></li><script>
const languagesMenuItem = document.getElementById("languages-menu");
const languagesDropDown = document.getElementById("languages-dropdown");
languagesMenuItem.addEventListener("click", function(event) {
event.preventDefault();
if (languagesDropDown.className == "hide") {
languagesDropDown.className = "visible";
} else {
languagesDropDown.className = "hide";
}
});
</script></span></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Pulsar SQL</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Get Started</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/getting-started-standalone">Run Pulsar locally</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/getting-started-docker">Run Pulsar in Docker</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/getting-started-helm">Run Pulsar in Kubernetes</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Concepts and Architecture</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/concepts-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/concepts-messaging">Messaging</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/concepts-architecture-overview">Architecture</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/concepts-clients">Clients</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/concepts-replication">Geo Replication</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/concepts-multi-tenancy">Multi Tenancy</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/concepts-authentication">Authentication and Authorization</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/concepts-topic-compaction">Topic Compaction</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/concepts-proxy-sni-routing">Proxy support with SNI routing</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/concepts-multiple-advertised-listeners">Multiple advertised listeners</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Pulsar Schema</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/schema-get-started">Get started</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/schema-understand">Understand schema</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/schema-evolution-compatibility">Schema evolution and compatibility</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/schema-manage">Manage schema</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Pulsar Functions</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/functions-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/functions-worker">Setup: Pulsar Functions Worker</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/functions-runtime">Setup: Configure Functions runtime</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/functions-develop">How-to: Develop</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/functions-package">How-to: Package</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/functions-debug">How-to: Debug</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/functions-deploy">How-to: Deploy</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/functions-cli">Reference: CLI</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/window-functions-context">Window Functions: Context</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Pulsar IO</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/io-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/io-quickstart">Get started</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/io-use">Use</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/io-debug">Debug</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/io-connectors">Built-in connector</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/io-cdc">CDC connector</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/io-develop">Develop</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/io-cli">CLI</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Pulsar SQL</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/sql-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/sql-getting-started">Query data</a></li><li class="navListItem navListItemActive"><a class="navItem" href="/docs/en/2.7.3/sql-deployment-configurations">Configuration and deployment</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/sql-rest-api">REST APIs</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Tiered Storage</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/tiered-storage-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/tiered-storage-aws">AWS S3 offloader</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/tiered-storage-gcs">GCS offloader</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/tiered-storage-filesystem">Filesystem offloader</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/tiered-storage-azure">Azure BlobStore offloader</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Transactions</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/transactions">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/transactions-guarantee">Transactions Guarantee</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/transactions-api">Transactions API</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Kubernetes (Helm)</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/helm-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/helm-prepare">Prepare</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/helm-install">Install</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/helm-deploy">Deployment</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/helm-upgrade">Upgrade</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/helm-tools">Required Tools</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Deployment</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/deploy-aws">Amazon Web Services</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/deploy-kubernetes">Kubernetes</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/deploy-bare-metal">Bare metal</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/deploy-bare-metal-multi-cluster">Bare metal multi-cluster</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/deploy-docker">Docker</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/deploy-monitoring">Monitor</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Administration</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/administration-zk-bk">ZooKeeper and BookKeeper</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/administration-geo">Geo-replication</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/administration-pulsar-manager">Pulsar Manager</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/administration-stats">Pulsar statistics</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/administration-load-balance">Load balance</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/administration-proxy">Pulsar proxy</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/administration-upgrade">Upgrade</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/administration-isolation">Pulsar isolation</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Security</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/security-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/security-tls-transport">Transport Encryption using TLS</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/security-tls-authentication">Authentication using TLS</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/security-tls-keystore">Using TLS with KeyStore configure</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/security-jwt">Authentication using JWT</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/security-athenz">Authentication using Athenz</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/security-kerberos">Authentication using Kerberos</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/security-oauth2">Authentication using OAuth 2.0 access tokens</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/security-authorization">Authorization and ACLs</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/security-encryption">End-to-End Encryption</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/security-extending">Extending</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/security-bouncy-castle">Bouncy Castle Providers</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Performance</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/performance-pulsar-perf">Pulsar Perf</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Client Libraries</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/client-libraries">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/client-libraries-java">Java</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/client-libraries-go">Go</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/client-libraries-python">Python</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/client-libraries-cpp">C++</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/client-libraries-node">Node.js</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/client-libraries-websocket">WebSocket</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/client-libraries-dotnet">C#</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Admin API</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/admin-api-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/admin-api-clusters">Clusters</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/admin-api-tenants">Tenants</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/admin-api-brokers">Brokers</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/admin-api-namespaces">Namespaces</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/admin-api-permissions">Permissions</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/admin-api-topics">Topics</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/admin-api-functions">Functions</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Adaptors</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/adaptors-kafka">Kafka client wrapper</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/adaptors-spark">Apache Spark</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/adaptors-storm">Apache Storm</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Cookbooks</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/cookbooks-compaction">Topic compaction</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/cookbooks-deduplication">Message deduplication</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/cookbooks-non-persistent">Non-persistent messaging</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/cookbooks-retention-expiry">Message retention and expiry</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/cookbooks-encryption">Encryption</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/cookbooks-message-queue">Message queue</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/cookbooks-bookkeepermetadata">BookKeeper Ledger Metadata</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Development</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/develop-tools">Simulation tools</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/developing-binary-protocol">Binary protocol</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/develop-schema">Custom schema storage</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/develop-load-manager">Modular load manager</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/develop-cpp">Building Pulsar C++ client</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Reference</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/reference-terminology">Terminology</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/reference-cli-tools">Pulsar CLI tools</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/reference-configuration">Pulsar configuration</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.3/reference-metrics">Pulsar Metrics</a></li></ul></div></div></section></div><script>
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
var links = coll[i].nextElementSibling.getElementsByTagName('*');
if (checkActiveCategory){
for (var j = 0; j < links.length; j++) {
if (links[j].classList.contains('navListItemActive')){
coll[i].nextElementSibling.classList.toggle('hide');
coll[i].childNodes[1].classList.toggle('rotate');
checkActiveCategory = false;
break;
}
}
}
coll[i].addEventListener('click', function() {
var arrow = this.childNodes[1];
arrow.classList.toggle('rotate');
var content = this.nextElementSibling;
content.classList.toggle('hide');
});
}
document.addEventListener('DOMContentLoaded', function() {
createToggler('#navToggler', '#docsNav', 'docsSliderActive');
createToggler('#tocToggler', 'body', 'tocActive');
var headings = document.querySelector('.toc-headings');
headings && headings.addEventListener('click', function(event) {
var el = event.target;
while(el !== headings){
if (el.tagName === 'A') {
document.body.classList.remove('tocActive');
break;
} else{
el = el.parentNode;
}
}
}, false);
function createToggler(togglerSelector, targetSelector, className) {
var toggler = document.querySelector(togglerSelector);
var target = document.querySelector(targetSelector);
if (!toggler) {
return;
}
toggler.onclick = function(event) {
event.preventDefault();
target.classList.toggle(className);
};
}
});
</script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/pulsar/edit/master/site2/docs/sql-deployment-configurations.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Pulsar SQL configuration and deployment</h1></header><article><div><span><p>You can configure Presto Pulsar connector and deploy a cluster with the following instruction.</p>
<h2><a class="anchor" aria-hidden="true" id="configure-presto-pulsar-connector"></a><a href="#configure-presto-pulsar-connector" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Configure Presto Pulsar Connector</h2>
<p>You can configure Presto Pulsar Connector in the <code>${project.root}/conf/presto/catalog/pulsar.properties</code> properties file. The configuration for the connector and the default values are as follows.</p>
<pre><code class="hljs css language-properties"><span class="hljs-comment"># name of the connector to be displayed in the catalog</span>
<span class="hljs-meta">connector.name</span>=<span class="hljs-string">pulsar</span>
<span class="hljs-comment">
# the url of Pulsar broker service</span>
<span class="hljs-meta">pulsar.broker-service-url</span>=<span class="hljs-string">http://localhost:8080</span>
<span class="hljs-comment">
# URI of Zookeeper cluster</span>
<span class="hljs-meta">pulsar.zookeeper-uri</span>=<span class="hljs-string">localhost:2181</span>
<span class="hljs-comment">
# minimum number of entries to read at a single time</span>
<span class="hljs-meta">pulsar.entry-read-batch-size</span>=<span class="hljs-string">100</span>
<span class="hljs-comment">
# default number of splits to use per query</span>
<span class="hljs-meta">pulsar.target-num-splits</span>=<span class="hljs-string">4</span>
</code></pre>
<p>You can connect Presto to a Pulsar cluster with multiple hosts. To configure multiple hosts for brokers, add multiple URLs to <code>pulsar.broker-service-url</code>. To configure multiple hosts for ZooKeeper, add multiple URIs to <code>pulsar.zookeeper-uri</code>. The following is an example.</p>
<pre><code class="hljs"><span class="hljs-attr">pulsar.broker-service-url</span>=http://localhost:<span class="hljs-number">8080</span>,localhost:<span class="hljs-number">8081</span>,localhost:<span class="hljs-number">8082</span>
<span class="hljs-attr">pulsar.zookeeper-uri</span>=localhost1,localhost2:<span class="hljs-number">2181</span>
</code></pre>
<p><strong>Note: by default, Pulsar SQL does not get the last message in a topic</strong>. It is by design and controlled by settings. By default, BookKeeper LAC only advances when subsequent entries are added. If there is no subsequent entry added, the last written entry is not visible to readers until the ledger is closed. This is not a problem for Pulsar which uses managed ledger, but Pulsar SQL directly reads from BookKeeper ledger.</p>
<p>If you want to get the last message in a topic, set the following configurations:</p>
<ol>
<li><p>For the broker configuration, set <code>bookkeeperExplicitLacIntervalInMills</code> &gt; 0 in <code>broker.conf</code> or <code>standalone.conf</code>.</p></li>
<li><p>For the Presto configuration, set <code>pulsar.bookkeeper-explicit-interval</code> &gt; 0 and <code>pulsar.bookkeeper-use-v2-protocol=false</code>.</p></li>
</ol>
<p>However, using BookKeeper V3 protocol introduces additional GC overhead to BK as it uses Protobuf.</p>
<h2><a class="anchor" aria-hidden="true" id="query-data-from-existing-presto-clusters"></a><a href="#query-data-from-existing-presto-clusters" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Query data from existing Presto clusters</h2>
<p>If you already have a Presto cluster, you can copy the Presto Pulsar connector plugin to your existing cluster. Download the archived plugin package with the following command.</p>
<pre><code class="hljs css language-bash">$ wget https://archive.apache.org/dist/pulsar/pulsar-2.7.3/apache-pulsar-2.7.3-bin.tar.gz
</code></pre>
<h2><a class="anchor" aria-hidden="true" id="deploy-a-new-cluster"></a><a href="#deploy-a-new-cluster" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Deploy a new cluster</h2>
<p>Since Pulsar SQL is powered by <a href="https://prestosql.io">Presto</a>, the configuration for deployment is the same for the Pulsar SQL worker.</p>
<blockquote>
<p>Note<br>
For how to set up a standalone single node environment, refer to <a href="/docs/en/2.7.3/sql-getting-started">Query data</a>.</p>
</blockquote>
<p>You can use the same CLI args as the Presto launcher.</p>
<pre><code class="hljs css language-bash">$ ./bin/pulsar sql-worker --<span class="hljs-built_in">help</span>
Usage: launcher [options] <span class="hljs-built_in">command</span>
Commands: run, start, stop, restart, <span class="hljs-built_in">kill</span>, status
Options:
-h, --<span class="hljs-built_in">help</span> show this <span class="hljs-built_in">help</span> message and <span class="hljs-built_in">exit</span>
-v, --verbose Run verbosely
--etc-dir=DIR Defaults to INSTALL_PATH/etc
--launcher-config=FILE
Defaults to INSTALL_PATH/bin/launcher.properties
--node-config=FILE Defaults to ETC_DIR/node.properties
--jvm-config=FILE Defaults to ETC_DIR/jvm.config
--config=FILE Defaults to ETC_DIR/config.properties
--<span class="hljs-built_in">log</span>-levels-file=FILE
Defaults to ETC_DIR/log.properties
--data-dir=DIR Defaults to INSTALL_PATH
--pid-file=FILE Defaults to DATA_DIR/var/run/launcher.pid
--launcher-log-file=FILE
Defaults to DATA_DIR/var/<span class="hljs-built_in">log</span>/launcher.log (only <span class="hljs-keyword">in</span>
daemon mode)
--server-log-file=FILE
Defaults to DATA_DIR/var/<span class="hljs-built_in">log</span>/server.log (only <span class="hljs-keyword">in</span>
daemon mode)
-D NAME=VALUE Set a Java system property
</code></pre>
<p>The default configuration for the cluster is located in <code>${project.root}/conf/presto</code>. You can customize your deployment by modifying the default configuration.</p>
<p>You can set the worker to read from a different configuration directory, or set a different directory to write data.</p>
<pre><code class="hljs css language-bash">$ ./bin/pulsar sql-worker run --etc-dir /tmp/incubator-pulsar/conf/presto --data-dir /tmp/presto-1
</code></pre>
<p>You can start the worker as daemon process.</p>
<pre><code class="hljs css language-bash">$ ./bin/pulsar sql-worker start
</code></pre>
<h3><a class="anchor" aria-hidden="true" id="deploy-a-cluster-on-multiple-nodes"></a><a href="#deploy-a-cluster-on-multiple-nodes" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Deploy a cluster on multiple nodes</h3>
<p>You can deploy a Pulsar SQL cluster or Presto cluster on multiple nodes. The following example shows how to deploy a cluster on three-node cluster.</p>
<ol>
<li>Copy the Pulsar binary distribution to three nodes.</li>
</ol>
<p>The first node runs as Presto coordinator. The minimal configuration requirement in the <code>${project.root}/conf/presto/config.properties</code> file is as follows.</p>
<pre><code class="hljs css language-properties"><span class="hljs-attr">coordinator</span>=<span class="hljs-string">true</span>
<span class="hljs-meta">node-scheduler.include-coordinator</span>=<span class="hljs-string">true</span>
<span class="hljs-meta">http-server.http.port</span>=<span class="hljs-string">8080</span>
<span class="hljs-meta">query.max-memory</span>=<span class="hljs-string">50GB</span>
<span class="hljs-meta">query.max-memory-per-node</span>=<span class="hljs-string">1GB</span>
<span class="hljs-meta">discovery-server.enabled</span>=<span class="hljs-string">true</span>
<span class="hljs-meta">discovery.uri</span>=<span class="hljs-string">&lt;coordinator-url&gt;</span>
</code></pre>
<p>The other two nodes serve as worker nodes, you can use the following configuration for worker nodes.</p>
<pre><code class="hljs css language-properties"><span class="hljs-attr">coordinator</span>=<span class="hljs-string">false</span>
<span class="hljs-meta">http-server.http.port</span>=<span class="hljs-string">8080</span>
<span class="hljs-meta">query.max-memory</span>=<span class="hljs-string">50GB</span>
<span class="hljs-meta">query.max-memory-per-node</span>=<span class="hljs-string">1GB</span>
<span class="hljs-meta">discovery.uri</span>=<span class="hljs-string">&lt;coordinator-url&gt;</span>
</code></pre>
<ol start="2">
<li><p>Modify <code>pulsar.broker-service-url</code> and <code>pulsar.zookeeper-uri</code> configuration in the <code>${project.root}/conf/presto/catalog/pulsar.properties</code> file accordingly for the three nodes.</p></li>
<li><p>Start the coordinator node.</p></li>
</ol>
<pre><code class="hljs">$ ./bin/pulsar sql-worker <span class="hljs-keyword">run</span><span class="bash">
</span></code></pre>
<ol start="4">
<li>Start worker nodes.</li>
</ol>
<pre><code class="hljs">$ ./bin/pulsar sql-worker <span class="hljs-keyword">run</span><span class="bash">
</span></code></pre>
<ol start="5">
<li>Start the SQL CLI and check the status of your cluster.</li>
</ol>
<pre><code class="hljs css language-bash">$ ./bin/pulsar sql --server &lt;coordinate_url&gt;
</code></pre>
<ol start="6">
<li>Check the status of your nodes.</li>
</ol>
<pre><code class="hljs css language-bash">presto&gt; SELECT * FROM system.runtime.nodes;
node_id | http_uri | node_version | coordinator | state
---------+-------------------------+--------------+-------------+--------
1 | http://192.168.2.1:8081 | testversion | <span class="hljs-literal">true</span> | active
3 | http://192.168.2.2:8081 | testversion | <span class="hljs-literal">false</span> | active
2 | http://192.168.2.3:8081 | testversion | <span class="hljs-literal">false</span> | active
</code></pre>
<p>For more information about deployment in Presto, refer to <a href="https://prestosql.io/docs/current/installation/deployment.html">Presto deployment</a>.</p>
<blockquote>
<p>Note<br>
The broker does not advance LAC, so when Pulsar SQL bypass broker to query data, it can only read entries up to the LAC that all the bookies learned. You can enable periodically write LAC on the broker by setting &quot;bookkeeperExplicitLacIntervalInMills&quot; in the broker.conf.</p>
</blockquote>
</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/en/2.7.3/sql-getting-started"><span class="arrow-prev">← </span><span>Query data</span></a><a class="docs-next button" href="/docs/en/2.7.3/sql-rest-api"><span>REST APIs</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#configure-presto-pulsar-connector">Configure Presto Pulsar Connector</a></li><li><a href="#query-data-from-existing-presto-clusters">Query data from existing Presto clusters</a></li><li><a href="#deploy-a-new-cluster">Deploy a new cluster</a><ul class="toc-headings"><li><a href="#deploy-a-cluster-on-multiple-nodes">Deploy a cluster on multiple nodes</a></li></ul></li></ul></nav></div><footer class="nav-footer" id="footer"><section class="copyright">Copyright © 2022 The Apache Software Foundation. All Rights Reserved. Apache, Apache Pulsar and the Apache feather logo are trademarks of The Apache Software Foundation.</section><span><script>
const community = document.querySelector("a[href='#community']").parentNode;
const communityMenu =
'<li>' +
'<a id="community-menu" href="#">Community <span style="font-size: 0.75em">&nbsp;▼</span></a>' +
'<div id="community-dropdown" class="hide">' +
'<ul id="community-dropdown-items">' +
'<li><a href="/en/contact">Contact</a></li>' +
'<li><a href="/en/contributing">Contributing</a></li>' +
'<li><a href="/en/coding-guide">Coding guide</a></li>' +
'<li><a href="/en/events">Events</a></li>' +
'<li><a href="https://twitter.com/Apache_Pulsar" target="_blank">Twitter &#x2750</a></li>' +
'<li><a href="https://github.com/apache/pulsar/wiki" target="_blank">Wiki &#x2750</a></li>' +
'<li><a href="https://github.com/apache/pulsar/issues" target="_blank">Issue tracking &#x2750</a></li>' +
'<li><a href="https://pulsar-summit.org/" target="_blank">Pulsar Summit &#x2750</a></li>' +
'<li>&nbsp;</li>' +
'<li><a href="/en/resources">Resources</a></li>' +
'<li><a href="/en/team">Team</a></li>' +
'<li><a href="/en/powered-by">Powered By</a></li>' +
'</ul>' +
'</div>' +
'</li>';
community.innerHTML = communityMenu;
const communityMenuItem = document.getElementById("community-menu");
const communityDropDown = document.getElementById("community-dropdown");
communityMenuItem.addEventListener("click", function(event) {
event.preventDefault();
if (communityDropDown.className == 'hide') {
communityDropDown.className = 'visible';
} else {
communityDropDown.className = 'hide';
}
});
</script></span></footer></div><script>window.twttr=(function(d,s, id){var js,fjs=d.getElementsByTagName(s)[0],t=window.twttr||{};if(d.getElementById(id))return t;js=d.createElement(s);js.id=id;js.src='https://platform.twitter.com/widgets.js';fjs.parentNode.insertBefore(js, fjs);t._e = [];t.ready = function(f) {t._e.push(f);};return t;}(document, 'script', 'twitter-wjs'));</script></body></html>