Google Git
Sign in
apache / pulsar-site / aee2d9089b6b8e327f04261353077211f4abd636 / . / content / docs / en / sql-deployment-configurations / index.html
blob: f17b4ef8bd3728b50f469135b54f000eac48feae [file] [log] [blame]
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Pulsar SQL configuration and deployment · Apache Pulsar</title><meta name="viewport" content="width=device-width, initial-scale=1.0"/><meta name="generator" content="Docusaurus"/><meta name="description" content="You can configure Presto Pulsar connector and deploy a cluster with the following instruction."/><meta name="docsearch:version" content="2.10.0"/><meta name="docsearch:language" content="en"/><meta property="og:title" content="Pulsar SQL configuration and deployment · Apache Pulsar"/><meta property="og:type" content="website"/><meta property="og:url" content="https://pulsar.apache.org/"/><meta property="og:description" content="You can configure Presto Pulsar connector and deploy a cluster with the following instruction."/><meta name="twitter:card" content="summary"/><meta name="twitter:image" content="https://pulsar.apache.org/img/pulsar.svg"/><link rel="shortcut icon" href="/img/pulsar.ico"/><link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/atom-one-dark.min.css"/><link rel="alternate" type="application/atom+xml" href="https://pulsar.apache.org/blog/atom.xml" title="Apache Pulsar Blog ATOM Feed"/><link rel="alternate" type="application/rss+xml" href="https://pulsar.apache.org/blog/feed.xml" title="Apache Pulsar Blog RSS Feed"/><link rel="stylesheet" href="/css/code-blocks-buttons.css"/><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.0/clipboard.min.js"></script><script type="text/javascript" src="/js/custom.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/en"><img class="logo" src="/img/pulsar.svg" alt="Apache Pulsar"/></a><a href="/en/versions"><h3>2.10.0</h3></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive"><a href="/docs/en/getting-started-standalone" target="_self">Docs</a></li><li class=""><a href="/en/download" target="_self">Download</a></li><li class="siteNavGroupActive"><a href="/docs/en/client-libraries" target="_self">Clients</a></li><li class=""><a href="#restapis" target="_self">REST APIs</a></li><li class=""><a href="#cli" target="_self">Cli</a></li><li class=""><a href="/blog/" target="_self">Blog</a></li><li class=""><a href="#community" target="_self">Community</a></li><li class=""><a href="#apache" target="_self">Apache</a></li><li class=""><a href="https://pulsar-next.staged.apache.org/" target="_self">New Website (Beta)</a></li><span><li><a id="languages-menu" href="#"><img class="languages-icon" src="/img/language.svg" alt="Languages icon"/>English</a><div id="languages-dropdown" class="hide"><ul id="languages-dropdown-items"><li><a href="/docs/ja/sql-deployment-configurations">日本語</a></li><li><a href="/docs/fr/sql-deployment-configurations">Français</a></li><li><a href="/docs/ko/sql-deployment-configurations">한국어</a></li><li><a href="/docs/zh-CN/sql-deployment-configurations">中文</a></li><li><a href="/docs/zh-TW/sql-deployment-configurations">繁體中文</a></li><li><a href="https://crowdin.com/project/apache-pulsar" target="_blank" rel="noreferrer noopener">Help Translate</a></li></ul></div></li><script>
const languagesMenuItem = document.getElementById("languages-menu");
const languagesDropDown = document.getElementById("languages-dropdown");
languagesMenuItem.addEventListener("click", function(event) {
event.preventDefault();
if (languagesDropDown.className == "hide") {
languagesDropDown.className = "visible";
} else {
languagesDropDown.className = "hide";
}
});
</script></span></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Pulsar SQL</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Get Started</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/getting-started-standalone">Run Pulsar locally</a></li><li class="navListItem"><a class="navItem" href="/docs/en/getting-started-docker">Run Pulsar in Docker</a></li><li class="navListItem"><a class="navItem" href="/docs/en/getting-started-helm">Run Pulsar in Kubernetes</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Concepts and Architecture</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/concepts-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/concepts-messaging">Messaging</a></li><li class="navListItem"><a class="navItem" href="/docs/en/concepts-architecture-overview">Architecture</a></li><li class="navListItem"><a class="navItem" href="/docs/en/concepts-clients">Clients</a></li><li class="navListItem"><a class="navItem" href="/docs/en/concepts-replication">Geo Replication</a></li><li class="navListItem"><a class="navItem" href="/docs/en/concepts-multi-tenancy">Multi Tenancy</a></li><li class="navListItem"><a class="navItem" href="/docs/en/concepts-authentication">Authentication and Authorization</a></li><li class="navListItem"><a class="navItem" href="/docs/en/concepts-topic-compaction">Topic Compaction</a></li><li class="navListItem"><a class="navItem" href="/docs/en/concepts-proxy-sni-routing">Proxy support with SNI routing</a></li><li class="navListItem"><a class="navItem" href="/docs/en/concepts-multiple-advertised-listeners">Multiple advertised listeners</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Pulsar Schema</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/schema-get-started">Get started</a></li><li class="navListItem"><a class="navItem" href="/docs/en/schema-understand">Understand schema</a></li><li class="navListItem"><a class="navItem" href="/docs/en/schema-evolution-compatibility">Schema evolution and compatibility</a></li><li class="navListItem"><a class="navItem" href="/docs/en/schema-manage">Manage schema</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Pulsar Functions</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/functions-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/functions-runtime">Setup: Configure Functions runtime</a></li><li class="navListItem"><a class="navItem" href="/docs/en/functions-worker">Setup: Pulsar Functions Worker</a></li><li class="navListItem"><a class="navItem" href="/docs/en/functions-develop">How-to: Develop</a></li><li class="navListItem"><a class="navItem" href="/docs/en/functions-package">How-to: Package</a></li><li class="navListItem"><a class="navItem" href="/docs/en/functions-debug">How-to: Debug</a></li><li class="navListItem"><a class="navItem" href="/docs/en/functions-deploy">How-to: Deploy</a></li><li class="navListItem"><a class="navItem" href="/docs/en/functions-cli">Reference: CLI</a></li><li class="navListItem"><a class="navItem" href="/docs/en/window-functions-context">Window Functions: Context</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Pulsar IO</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/io-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/io-quickstart">Get started</a></li><li class="navListItem"><a class="navItem" href="/docs/en/io-use">Use</a></li><li class="navListItem"><a class="navItem" href="/docs/en/io-debug">Debug</a></li><li class="navListItem"><a class="navItem" href="/docs/en/io-connectors">Built-in connector</a></li><li class="navListItem"><a class="navItem" href="/docs/en/io-cdc">CDC connector</a></li><li class="navListItem"><a class="navItem" href="/docs/en/io-develop">Develop</a></li><li class="navListItem"><a class="navItem" href="/docs/en/io-cli">CLI</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Pulsar SQL</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/sql-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/sql-getting-started">Query data</a></li><li class="navListItem navListItemActive"><a class="navItem" href="/docs/en/sql-deployment-configurations">Configuration and deployment</a></li><li class="navListItem"><a class="navItem" href="/docs/en/sql-rest-api">REST APIs</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Tiered Storage</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/tiered-storage-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/tiered-storage-aws">AWS S3 offloader</a></li><li class="navListItem"><a class="navItem" href="/docs/en/tiered-storage-gcs">GCS offloader</a></li><li class="navListItem"><a class="navItem" href="/docs/en/tiered-storage-filesystem">Filesystem offloader</a></li><li class="navListItem"><a class="navItem" href="/docs/en/tiered-storage-azure">Azure BlobStore offloader</a></li><li class="navListItem"><a class="navItem" href="/docs/en/tiered-storage-aliyun">Aliyun OSS offloader</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Transactions</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/txn-why">Why transactions?</a></li><li class="navListItem"><a class="navItem" href="/docs/en/txn-what">What are transactions?</a></li><li class="navListItem"><a class="navItem" href="/docs/en/txn-how">How transactions work?</a></li><li class="navListItem"><a class="navItem" href="/docs/en/txn-use">How to use transactions?</a></li><li class="navListItem"><a class="navItem" href="/docs/en/txn-monitor">How to monitor transactions?</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Kubernetes (Helm)</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/helm-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/helm-prepare">Prepare</a></li><li class="navListItem"><a class="navItem" href="/docs/en/helm-install">Install</a></li><li class="navListItem"><a class="navItem" href="/docs/en/helm-deploy">Deployment</a></li><li class="navListItem"><a class="navItem" href="/docs/en/helm-upgrade">Upgrade</a></li><li class="navListItem"><a class="navItem" href="/docs/en/helm-tools">Required Tools</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Deployment</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/deploy-aws">Amazon Web Services</a></li><li class="navListItem"><a class="navItem" href="/docs/en/deploy-kubernetes">Kubernetes</a></li><li class="navListItem"><a class="navItem" href="/docs/en/deploy-bare-metal">Bare metal</a></li><li class="navListItem"><a class="navItem" href="/docs/en/deploy-bare-metal-multi-cluster">Bare metal multi-cluster</a></li><li class="navListItem"><a class="navItem" href="/docs/en/deploy-dcos">DC/OS</a></li><li class="navListItem"><a class="navItem" href="/docs/en/deploy-docker">Docker</a></li><li class="navListItem"><a class="navItem" href="/docs/en/deploy-monitoring">Monitor</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Administration</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/administration-zk-bk">ZooKeeper and BookKeeper</a></li><li class="navListItem"><a class="navItem" href="/docs/en/administration-geo">Geo-replication</a></li><li class="navListItem"><a class="navItem" href="/docs/en/administration-pulsar-manager">Pulsar Manager</a></li><li class="navListItem"><a class="navItem" href="/docs/en/administration-stats">Pulsar statistics</a></li><li class="navListItem"><a class="navItem" href="/docs/en/administration-load-balance">Load balance</a></li><li class="navListItem"><a class="navItem" href="/docs/en/administration-proxy">Pulsar proxy</a></li><li class="navListItem"><a class="navItem" href="/docs/en/administration-upgrade">Upgrade</a></li><li class="navListItem"><a class="navItem" href="/docs/en/administration-isolation">Pulsar isolation</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Security</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/security-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/security-policy-and-supported-versions">Security Policy and Supported Versions</a></li><li class="navListItem"><a class="navItem" href="/docs/en/security-tls-transport">Transport Encryption using TLS</a></li><li class="navListItem"><a class="navItem" href="/docs/en/security-tls-authentication">Authentication using TLS</a></li><li class="navListItem"><a class="navItem" href="/docs/en/security-tls-keystore">Using TLS with KeyStore configure</a></li><li class="navListItem"><a class="navItem" href="/docs/en/security-jwt">Authentication using JWT</a></li><li class="navListItem"><a class="navItem" href="/docs/en/security-athenz">Authentication using Athenz</a></li><li class="navListItem"><a class="navItem" href="/docs/en/security-kerberos">Authentication using Kerberos</a></li><li class="navListItem"><a class="navItem" href="/docs/en/security-oauth2">Authentication using OAuth 2.0 access tokens</a></li><li class="navListItem"><a class="navItem" href="/docs/en/security-authorization">Authorization and ACLs</a></li><li class="navListItem"><a class="navItem" href="/docs/en/security-encryption">End-to-End Encryption</a></li><li class="navListItem"><a class="navItem" href="/docs/en/security-extending">Extend Authentication and Authorization</a></li><li class="navListItem"><a class="navItem" href="/docs/en/security-bouncy-castle">Bouncy Castle Providers</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Performance</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/performance-pulsar-perf">Pulsar Perf</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Client Libraries</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/client-libraries">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/client-libraries-java">Java</a></li><li class="navListItem"><a class="navItem" href="/docs/en/client-libraries-go">Go</a></li><li class="navListItem"><a class="navItem" href="/docs/en/client-libraries-python">Python</a></li><li class="navListItem"><a class="navItem" href="/docs/en/client-libraries-cpp">C++</a></li><li class="navListItem"><a class="navItem" href="/docs/en/client-libraries-node">Node.js</a></li><li class="navListItem"><a class="navItem" href="/docs/en/client-libraries-websocket">WebSocket</a></li><li class="navListItem"><a class="navItem" href="/docs/en/client-libraries-dotnet">C#</a></li><li class="navListItem"><a class="navItem" href="/docs/en/client-libraries-rest">REST</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Admin API</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/admin-api-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/admin-api-clusters">Clusters</a></li><li class="navListItem"><a class="navItem" href="/docs/en/admin-api-tenants">Tenants</a></li><li class="navListItem"><a class="navItem" href="/docs/en/admin-api-brokers">Brokers</a></li><li class="navListItem"><a class="navItem" href="/docs/en/admin-api-namespaces">Namespaces</a></li><li class="navListItem"><a class="navItem" href="/docs/en/admin-api-permissions">Permissions</a></li><li class="navListItem"><a class="navItem" href="/docs/en/admin-api-topics">Topics</a></li><li class="navListItem"><a class="navItem" href="/docs/en/admin-api-functions">Functions</a></li><li class="navListItem"><a class="navItem" href="/docs/en/admin-api-packages">Packages</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Adaptors</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/adaptors-kafka">Kafka client wrapper</a></li><li class="navListItem"><a class="navItem" href="/docs/en/adaptors-spark">Apache Spark</a></li><li class="navListItem"><a class="navItem" href="/docs/en/adaptors-storm">Apache Storm</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Cookbooks</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/cookbooks-compaction">Topic compaction</a></li><li class="navListItem"><a class="navItem" href="/docs/en/cookbooks-deduplication">Message deduplication</a></li><li class="navListItem"><a class="navItem" href="/docs/en/cookbooks-non-persistent">Non-persistent messaging</a></li><li class="navListItem"><a class="navItem" href="/docs/en/cookbooks-retention-expiry">Message retention and expiry</a></li><li class="navListItem"><a class="navItem" href="/docs/en/cookbooks-encryption">Encryption</a></li><li class="navListItem"><a class="navItem" href="/docs/en/cookbooks-message-queue">Message queue</a></li><li class="navListItem"><a class="navItem" href="/docs/en/cookbooks-bookkeepermetadata">BookKeeper Ledger Metadata</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Development</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/develop-tools">Simulation tools</a></li><li class="navListItem"><a class="navItem" href="/docs/en/developing-binary-protocol">Binary protocol</a></li><li class="navListItem"><a class="navItem" href="/docs/en/develop-schema">Custom schema storage</a></li><li class="navListItem"><a class="navItem" href="/docs/en/develop-load-manager">Modular load manager</a></li><li class="navListItem"><a class="navItem" href="/docs/en/develop-plugin">Plugin</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Reference</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/reference-terminology">Terminology</a></li><li class="navListItem"><a class="navItem" href="/docs/en/reference-cli-tools">Pulsar CLI tools</a></li><li class="navListItem"><a class="navItem" href="/docs/en/reference-configuration">Pulsar configuration</a></li><li class="navListItem"><a class="navItem" href="/docs/en/reference-metrics">Pulsar Metrics</a></li></ul></div></div></section></div><script>
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
var links = coll[i].nextElementSibling.getElementsByTagName('*');
if (checkActiveCategory){
for (var j = 0; j < links.length; j++) {
if (links[j].classList.contains('navListItemActive')){
coll[i].nextElementSibling.classList.toggle('hide');
coll[i].childNodes[1].classList.toggle('rotate');
checkActiveCategory = false;
break;
}
}
}
coll[i].addEventListener('click', function() {
var arrow = this.childNodes[1];
arrow.classList.toggle('rotate');
var content = this.nextElementSibling;
content.classList.toggle('hide');
});
}
document.addEventListener('DOMContentLoaded', function() {
createToggler('#navToggler', '#docsNav', 'docsSliderActive');
createToggler('#tocToggler', 'body', 'tocActive');
var headings = document.querySelector('.toc-headings');
headings && headings.addEventListener('click', function(event) {
var el = event.target;
while(el !== headings){
if (el.tagName === 'A') {
document.body.classList.remove('tocActive');
break;
} else{
el = el.parentNode;
}
}
}, false);
function createToggler(togglerSelector, targetSelector, className) {
var toggler = document.querySelector(togglerSelector);
var target = document.querySelector(targetSelector);
if (!toggler) {
return;
}
toggler.onclick = function(event) {
event.preventDefault();
target.classList.toggle(className);
};
}
});
</script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/pulsar/edit/master/site2/docs/sql-deployment-configurations.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Pulsar SQL configuration and deployment</h1></header><article><div><span><p>You can configure Presto Pulsar connector and deploy a cluster with the following instruction.</p>
<h2><a class="anchor" aria-hidden="true" id="configure-presto-pulsar-connector"></a><a href="#configure-presto-pulsar-connector" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Configure Presto Pulsar Connector</h2>
<p>You can configure Presto Pulsar Connector in the <code>${project.root}/conf/presto/catalog/pulsar.properties</code> properties file. The configuration for the connector and the default values are as follows.</p>
<pre><code class="hljs css language-properties"><span class="hljs-comment"># name of the connector to be displayed in the catalog</span>
<span class="hljs-meta">connector.name</span>=<span class="hljs-string">pulsar</span>
<span class="hljs-comment">
# the url of Pulsar broker service</span>
<span class="hljs-meta">pulsar.web-service-url</span>=<span class="hljs-string">http://localhost:8080</span>
<span class="hljs-comment">
# URI of Zookeeper cluster</span>
<span class="hljs-meta">pulsar.zookeeper-uri</span>=<span class="hljs-string">localhost:2181</span>
<span class="hljs-comment">
# minimum number of entries to read at a single time</span>
<span class="hljs-meta">pulsar.entry-read-batch-size</span>=<span class="hljs-string">100</span>
<span class="hljs-comment">
# default number of splits to use per query</span>
<span class="hljs-meta">pulsar.target-num-splits</span>=<span class="hljs-string">4</span>
<span class="hljs-comment">
# max size of one batch message (default value is 5MB)</span>
<span class="hljs-meta">pulsar.max-message-size</span>=<span class="hljs-string">5242880</span>
<span class="hljs-comment">
# number of split used when querying data from pulsar</span>
<span class="hljs-meta">pulsar.target-num-splits</span>=<span class="hljs-string">2</span>
<span class="hljs-comment">
# size of queue to buffer entry read from pulsar</span>
<span class="hljs-meta">pulsar.max-split-entry-queue-size</span>=<span class="hljs-string">1000</span>
<span class="hljs-comment">
# size of queue to buffer message extract from entries</span>
<span class="hljs-meta">pulsar.max-split-message-queue-size</span>=<span class="hljs-string">10000</span>
<span class="hljs-comment">
# status provider to record connector metrics</span>
<span class="hljs-meta">pulsar.stats-provider</span>=<span class="hljs-string">org.apache.bookkeeper.stats.NullStatsProvider</span>
<span class="hljs-comment">
# config in map format for stats provider e.g. {"key1":"val1","key2":"val2"}</span>
<span class="hljs-meta">pulsar.stats-provider-configs</span>=<span class="hljs-string">{}</span>
<span class="hljs-comment">
# whether to rewrite Pulsar's default topic delimiter '/'</span>
<span class="hljs-meta">pulsar.namespace-delimiter-rewrite-enable</span>=<span class="hljs-string">false</span>
<span class="hljs-comment">
# delimiter used to rewrite Pulsar's default delimiter '/', use if default is causing incompatibility with other system like Superset</span>
<span class="hljs-meta">pulsar.rewrite-namespace-delimiter</span>=<span class="hljs-string">“/”</span>
<span class="hljs-comment">
# maximum number of thread pool size for ledger offloader.</span>
<span class="hljs-meta">pulsar.managed-ledger-offload-max-threads</span>=<span class="hljs-string">2</span>
<span class="hljs-comment">
# driver used to offload or read cold data to or from long-term storage</span>
<span class="hljs-meta">pulsar.managed-ledger-offload-driver</span>=<span class="hljs-string">null</span>
<span class="hljs-comment">
# directory to load offloaders nar file.</span>
<span class="hljs-meta">pulsar.offloaders-directory</span>=<span class="hljs-string">"./offloaders"</span>
<span class="hljs-comment">
# properties and configurations related to specific offloader implementation as map e.g. {"key1":"val1","key2":"val2"}</span>
<span class="hljs-meta">pulsar.offloader-properties</span>=<span class="hljs-string">{}</span>
<span class="hljs-comment">
# authentication plugin used to authenticate to Pulsar cluster</span>
<span class="hljs-meta">pulsar.auth-plugin</span>=<span class="hljs-string">null</span>
<span class="hljs-comment">
# authentication parameter used to authenticate to the Pulsar cluster as a string e.g. "key1:val1,key2:val2".</span>
<span class="hljs-meta">pulsar.auth-params</span>=<span class="hljs-string">null</span>
<span class="hljs-comment">
# whether the Pulsar client accept an untrusted TLS certificate from broker</span>
<span class="hljs-meta">pulsar.tls-allow-insecure-connection</span>=<span class="hljs-string">null</span>
<span class="hljs-comment">
# whether to allow hostname verification when a client connects to broker over TLS.</span>
<span class="hljs-meta">pulsar.tls-hostname-verification-enable</span>=<span class="hljs-string">null</span>
<span class="hljs-comment">
# path for the trusted TLS certificate file of Pulsar broker</span>
<span class="hljs-meta">pulsar.tls-trust-cert-file-path</span>=<span class="hljs-string">null</span>
<span class="hljs-comment">
# set the threshold for BookKeeper request throttle, default is disabled</span>
<span class="hljs-meta">pulsar.bookkeeper-throttle-value</span>=<span class="hljs-string">0</span>
<span class="hljs-comment">
# set the number of IO thread</span>
<span class="hljs-meta">pulsar.bookkeeper-num-io-threads</span>=<span class="hljs-string">2 * Runtime.getRuntime().availableProcessors()</span>
<span class="hljs-comment">
# set the number of worker thread</span>
<span class="hljs-meta">pulsar.bookkeeper-num-worker-threads</span>=<span class="hljs-string">Runtime.getRuntime().availableProcessors()</span>
<span class="hljs-comment">
# whether to use BookKeeper V2 wire protocol</span>
<span class="hljs-meta">pulsar.bookkeeper-use-v2-protocol</span>=<span class="hljs-string">true</span>
<span class="hljs-comment">
# interval to check the need for sending an explicit LAC, default is disabled</span>
<span class="hljs-meta">pulsar.bookkeeper-explicit-interval</span>=<span class="hljs-string">0</span>
<span class="hljs-comment">
# size for managed ledger entry cache (in MB).</span>
<span class="hljs-meta">pulsar.managed-ledger-cache-size-MB</span>=<span class="hljs-string">0</span>
<span class="hljs-comment">
# number of threads to be used for managed ledger tasks dispatching</span>
<span class="hljs-meta">pulsar.managed-ledger-num-worker-threads</span>=<span class="hljs-string">Runtime.getRuntime().availableProcessors()</span>
<span class="hljs-comment">
# number of threads to be used for managed ledger scheduled tasks</span>
<span class="hljs-meta">pulsar.managed-ledger-num-scheduler-threads</span>=<span class="hljs-string">Runtime.getRuntime().availableProcessors()</span>
<span class="hljs-comment">
# directory used to store extraction NAR file</span>
<span class="hljs-meta">pulsar.nar-extraction-directory</span>=<span class="hljs-string">System.getProperty("java.io.tmpdir")</span>
</code></pre>
<p>You can connect Presto to a Pulsar cluster with multiple hosts. To configure multiple hosts for brokers, add multiple URLs to <code>pulsar.web-service-url</code>. To configure multiple hosts for ZooKeeper, add multiple URIs to <code>pulsar.zookeeper-uri</code>. The following is an example.</p>
<pre><code class="hljs"><span class="hljs-attr">pulsar.web-service-url</span>=http://localhost:<span class="hljs-number">8080</span>,localhost:<span class="hljs-number">8081</span>,localhost:<span class="hljs-number">8082</span>
<span class="hljs-attr">pulsar.zookeeper-uri</span>=localhost1,localhost2:<span class="hljs-number">2181</span>
</code></pre>
<p><strong>Note: by default, Pulsar SQL does not get the last message in a topic</strong>. It is by design and controlled by settings. By default, BookKeeper LAC only advances when subsequent entries are added. If there is no subsequent entry added, the last written entry is not visible to readers until the ledger is closed. This is not a problem for Pulsar which uses managed ledger, but Pulsar SQL directly reads from BookKeeper ledger.</p>
<p>If you want to get the last message in a topic, set the following configurations:</p>
<ol>
<li><p>For the broker configuration, set <code>bookkeeperExplicitLacIntervalInMills</code> &gt; 0 in <code>broker.conf</code> or <code>standalone.conf</code>.</p></li>
<li><p>For the Presto configuration, set <code>pulsar.bookkeeper-explicit-interval</code> &gt; 0 and <code>pulsar.bookkeeper-use-v2-protocol=false</code>.</p></li>
</ol>
<p>However, using BookKeeper V3 protocol introduces additional GC overhead to BK as it uses Protobuf.</p>
<h2><a class="anchor" aria-hidden="true" id="query-data-from-existing-presto-clusters"></a><a href="#query-data-from-existing-presto-clusters" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Query data from existing Presto clusters</h2>
<p>If you already have a Presto cluster, you can copy the Presto Pulsar connector plugin to your existing cluster. Download the archived plugin package with the following command.</p>
<pre><code class="hljs css language-bash">$ wget https://archive.apache.org/dist/pulsar/pulsar-2.10.0/apache-pulsar-2.10.0-bin.tar.gz
</code></pre>
<h2><a class="anchor" aria-hidden="true" id="deploy-a-new-cluster"></a><a href="#deploy-a-new-cluster" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Deploy a new cluster</h2>
<p>Since Pulsar SQL is powered by <a href="https://trino.io">Trino (formerly Presto SQL)</a>, the configuration for deployment is the same for the Pulsar SQL worker.</p>
<blockquote>
<p>Note<br>
For how to set up a standalone single node environment, refer to <a href="/docs/en/sql-getting-started">Query data</a>.</p>
</blockquote>
<p>You can use the same CLI args as the Presto launcher.</p>
<pre><code class="hljs css language-bash">$ ./bin/pulsar sql-worker --<span class="hljs-built_in">help</span>
Usage: launcher [options] <span class="hljs-built_in">command</span>
Commands: run, start, stop, restart, <span class="hljs-built_in">kill</span>, status
Options:
-h, --<span class="hljs-built_in">help</span> show this <span class="hljs-built_in">help</span> message and <span class="hljs-built_in">exit</span>
-v, --verbose Run verbosely
--etc-dir=DIR Defaults to INSTALL_PATH/etc
--launcher-config=FILE
Defaults to INSTALL_PATH/bin/launcher.properties
--node-config=FILE Defaults to ETC_DIR/node.properties
--jvm-config=FILE Defaults to ETC_DIR/jvm.config
--config=FILE Defaults to ETC_DIR/config.properties
--<span class="hljs-built_in">log</span>-levels-file=FILE
Defaults to ETC_DIR/log.properties
--data-dir=DIR Defaults to INSTALL_PATH
--pid-file=FILE Defaults to DATA_DIR/var/run/launcher.pid
--launcher-log-file=FILE
Defaults to DATA_DIR/var/<span class="hljs-built_in">log</span>/launcher.log (only <span class="hljs-keyword">in</span>
daemon mode)
--server-log-file=FILE
Defaults to DATA_DIR/var/<span class="hljs-built_in">log</span>/server.log (only <span class="hljs-keyword">in</span>
daemon mode)
-D NAME=VALUE Set a Java system property
</code></pre>
<p>The default configuration for the cluster is located in <code>${project.root}/conf/presto</code>. You can customize your deployment by modifying the default configuration.</p>
<p>You can set the worker to read from a different configuration directory, or set a different directory to write data.</p>
<pre><code class="hljs css language-bash">$ ./bin/pulsar sql-worker run --etc-dir /tmp/incubator-pulsar/conf/presto --data-dir /tmp/presto-1
</code></pre>
<p>You can start the worker as daemon process.</p>
<pre><code class="hljs css language-bash">$ ./bin/pulsar sql-worker start
</code></pre>
<h3><a class="anchor" aria-hidden="true" id="deploy-a-cluster-on-multiple-nodes"></a><a href="#deploy-a-cluster-on-multiple-nodes" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Deploy a cluster on multiple nodes</h3>
<p>You can deploy a Pulsar SQL cluster or Presto cluster on multiple nodes. The following example shows how to deploy a cluster on three-node cluster.</p>
<ol>
<li>Copy the Pulsar binary distribution to three nodes.</li>
</ol>
<p>The first node runs as Presto coordinator. The minimal configuration requirement in the <code>${project.root}/conf/presto/config.properties</code> file is as follows.</p>
<pre><code class="hljs css language-properties"><span class="hljs-attr">coordinator</span>=<span class="hljs-string">true</span>
<span class="hljs-meta">node-scheduler.include-coordinator</span>=<span class="hljs-string">true</span>
<span class="hljs-meta">http-server.http.port</span>=<span class="hljs-string">8080</span>
<span class="hljs-meta">query.max-memory</span>=<span class="hljs-string">50GB</span>
<span class="hljs-meta">query.max-memory-per-node</span>=<span class="hljs-string">1GB</span>
<span class="hljs-meta">discovery-server.enabled</span>=<span class="hljs-string">true</span>
<span class="hljs-meta">discovery.uri</span>=<span class="hljs-string">&lt;coordinator-url&gt;</span>
</code></pre>
<p>The other two nodes serve as worker nodes, you can use the following configuration for worker nodes.</p>
<pre><code class="hljs css language-properties"><span class="hljs-attr">coordinator</span>=<span class="hljs-string">false</span>
<span class="hljs-meta">http-server.http.port</span>=<span class="hljs-string">8080</span>
<span class="hljs-meta">query.max-memory</span>=<span class="hljs-string">50GB</span>
<span class="hljs-meta">query.max-memory-per-node</span>=<span class="hljs-string">1GB</span>
<span class="hljs-meta">discovery.uri</span>=<span class="hljs-string">&lt;coordinator-url&gt;</span>
</code></pre>
<ol start="2">
<li><p>Modify <code>pulsar.web-service-url</code> and <code>pulsar.zookeeper-uri</code> configuration in the <code>${project.root}/conf/presto/catalog/pulsar.properties</code> file accordingly for the three nodes.</p></li>
<li><p>Start the coordinator node.</p></li>
</ol>
<pre><code class="hljs">$ ./bin/pulsar sql-worker <span class="hljs-keyword">run</span><span class="bash">
</span></code></pre>
<ol start="4">
<li>Start worker nodes.</li>
</ol>
<pre><code class="hljs">$ ./bin/pulsar sql-worker <span class="hljs-keyword">run</span><span class="bash">
</span></code></pre>
<ol start="5">
<li>Start the SQL CLI and check the status of your cluster.</li>
</ol>
<pre><code class="hljs css language-bash">$ ./bin/pulsar sql --server &lt;coordinate_url&gt;
</code></pre>
<ol start="6">
<li>Check the status of your nodes.</li>
</ol>
<pre><code class="hljs css language-bash">presto&gt; SELECT * FROM system.runtime.nodes;
node_id | http_uri | node_version | coordinator | state
---------+-------------------------+--------------+-------------+--------
1 | http://192.168.2.1:8081 | testversion | <span class="hljs-literal">true</span> | active
3 | http://192.168.2.2:8081 | testversion | <span class="hljs-literal">false</span> | active
2 | http://192.168.2.3:8081 | testversion | <span class="hljs-literal">false</span> | active
</code></pre>
<p>For more information about deployment in Presto, refer to <a href="https://trino.io/docs/current/installation/deployment.html">Presto deployment</a>.</p>
<blockquote>
<p>Note<br>
The broker does not advance LAC, so when Pulsar SQL bypass broker to query data, it can only read entries up to the LAC that all the bookies learned. You can enable periodically write LAC on the broker by setting &quot;bookkeeperExplicitLacIntervalInMills&quot; in the broker.conf.</p>
</blockquote>
</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/en/sql-getting-started"><span class="arrow-prev">← </span><span>Query data</span></a><a class="docs-next button" href="/docs/en/sql-rest-api"><span>REST APIs</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#configure-presto-pulsar-connector">Configure Presto Pulsar Connector</a></li><li><a href="#query-data-from-existing-presto-clusters">Query data from existing Presto clusters</a></li><li><a href="#deploy-a-new-cluster">Deploy a new cluster</a><ul class="toc-headings"><li><a href="#deploy-a-cluster-on-multiple-nodes">Deploy a cluster on multiple nodes</a></li></ul></li></ul></nav></div><footer class="nav-footer" id="footer"><section class="copyright">Copyright © 2022 The Apache Software Foundation. All Rights Reserved. Apache, Apache Pulsar and the Apache feather logo are trademarks of The Apache Software Foundation.</section><span><script>
const community = document.querySelector("a[href='#community']").parentNode;
const communityMenu =
'<li>' +
'<a id="community-menu" href="#">Community <span style="font-size: 0.75em">&nbsp;▼</span></a>' +
'<div id="community-dropdown" class="hide">' +
'<ul id="community-dropdown-items">' +
'<li><a href="/en/contact">Contact</a></li>' +
'<li><a href="/en/contributing">Contributing</a></li>' +
'<li><a href="/en/coding-guide">Coding guide</a></li>' +
'<li><a href="/en/events">Events</a></li>' +
'<li><a href="https://twitter.com/Apache_Pulsar" target="_blank">Twitter &#x2750</a></li>' +
'<li><a href="https://github.com/apache/pulsar/wiki" target="_blank">Wiki &#x2750</a></li>' +
'<li><a href="https://github.com/apache/pulsar/issues" target="_blank">Issue tracking &#x2750</a></li>' +
'<li><a href="https://pulsar-summit.org/" target="_blank">Pulsar Summit &#x2750</a></li>' +
'<li>&nbsp;</li>' +
'<li><a href="/en/resources">Resources</a></li>' +
'<li><a href="/en/team">Team</a></li>' +
'<li><a href="/en/powered-by">Powered By</a></li>' +
'</ul>' +
'</div>' +
'</li>';
community.innerHTML = communityMenu;
const communityMenuItem = document.getElementById("community-menu");
const communityDropDown = document.getElementById("community-dropdown");
communityMenuItem.addEventListener("click", function(event) {
event.preventDefault();
if (communityDropDown.className == 'hide') {
communityDropDown.className = 'visible';
} else {
communityDropDown.className = 'hide';
}
});
</script></span></footer></div><script>window.twttr=(function(d,s, id){var js,fjs=d.getElementsByTagName(s)[0],t=window.twttr||{};if(d.getElementById(id))return t;js=d.createElement(s);js.id=id;js.src='https://platform.twitter.com/widgets.js';fjs.parentNode.insertBefore(js, fjs);t._e = [];t.ready = function(f) {t._e.push(f);};return t;}(document, 'script', 'twitter-wjs'));</script></body></html>