Google Git
Sign in
apache / pulsar-site / aee2d9089b6b8e327f04261353077211f4abd636 / . / content / docs / en / 2.7.2 / tiered-storage-filesystem.html
blob: ea20c13be6766666e937692ae087bf5fe4e3aee6 [file] [log] [blame]
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Use filesystem offloader with Pulsar · Apache Pulsar</title><meta name="viewport" content="width=device-width, initial-scale=1.0"/><meta name="generator" content="Docusaurus"/><meta name="description" content="This chapter guides you through every step of installing and configuring the filesystem offloader and using it with Pulsar."/><meta name="docsearch:version" content="2.7.2"/><meta name="docsearch:language" content="en"/><meta property="og:title" content="Use filesystem offloader with Pulsar · Apache Pulsar"/><meta property="og:type" content="website"/><meta property="og:url" content="https://pulsar.apache.org/"/><meta property="og:description" content="This chapter guides you through every step of installing and configuring the filesystem offloader and using it with Pulsar."/><meta name="twitter:card" content="summary"/><meta name="twitter:image" content="https://pulsar.apache.org/img/pulsar.svg"/><link rel="shortcut icon" href="/img/pulsar.ico"/><link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/atom-one-dark.min.css"/><link rel="alternate" type="application/atom+xml" href="https://pulsar.apache.org/blog/atom.xml" title="Apache Pulsar Blog ATOM Feed"/><link rel="alternate" type="application/rss+xml" href="https://pulsar.apache.org/blog/feed.xml" title="Apache Pulsar Blog RSS Feed"/><link rel="stylesheet" href="/css/code-blocks-buttons.css"/><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.0/clipboard.min.js"></script><script type="text/javascript" src="/js/custom.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/en"><img class="logo" src="/img/pulsar.svg" alt="Apache Pulsar"/></a><a href="/en/versions"><h3>2.7.2</h3></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive"><a href="/docs/en/2.7.2/getting-started-standalone" target="_self">Docs</a></li><li class=""><a href="/en/download" target="_self">Download</a></li><li class="siteNavGroupActive"><a href="/docs/en/2.7.2/client-libraries" target="_self">Clients</a></li><li class=""><a href="#restapis" target="_self">REST APIs</a></li><li class=""><a href="#cli" target="_self">Cli</a></li><li class=""><a href="/blog/" target="_self">Blog</a></li><li class=""><a href="#community" target="_self">Community</a></li><li class=""><a href="#apache" target="_self">Apache</a></li><li class=""><a href="https://pulsar-next.staged.apache.org/" target="_self">New Website (Beta)</a></li><span><li><a id="languages-menu" href="#"><img class="languages-icon" src="/img/language.svg" alt="Languages icon"/>English</a><div id="languages-dropdown" class="hide"><ul id="languages-dropdown-items"><li><a href="/docs/ja/2.7.2/tiered-storage-filesystem">日本語</a></li><li><a href="/docs/fr/2.7.2/tiered-storage-filesystem">Français</a></li><li><a href="/docs/ko/2.7.2/tiered-storage-filesystem">한국어</a></li><li><a href="/docs/zh-CN/2.7.2/tiered-storage-filesystem">中文</a></li><li><a href="/docs/zh-TW/2.7.2/tiered-storage-filesystem">繁體中文</a></li><li><a href="https://crowdin.com/project/apache-pulsar" target="_blank" rel="noreferrer noopener">Help Translate</a></li></ul></div></li><script>
const languagesMenuItem = document.getElementById("languages-menu");
const languagesDropDown = document.getElementById("languages-dropdown");
languagesMenuItem.addEventListener("click", function(event) {
event.preventDefault();
if (languagesDropDown.className == "hide") {
languagesDropDown.className = "visible";
} else {
languagesDropDown.className = "hide";
}
});
</script></span></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Tiered Storage</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Get Started</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/getting-started-standalone">Run Pulsar locally</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/getting-started-docker">Run Pulsar in Docker</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/getting-started-helm">Run Pulsar in Kubernetes</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Concepts and Architecture</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/concepts-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/concepts-messaging">Messaging</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/concepts-architecture-overview">Architecture</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/concepts-clients">Clients</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/concepts-replication">Geo Replication</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/concepts-multi-tenancy">Multi Tenancy</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/concepts-authentication">Authentication and Authorization</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/concepts-topic-compaction">Topic Compaction</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/concepts-proxy-sni-routing">Proxy support with SNI routing</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/concepts-multiple-advertised-listeners">Multiple advertised listeners</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Pulsar Schema</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/schema-get-started">Get started</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/schema-understand">Understand schema</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/schema-evolution-compatibility">Schema evolution and compatibility</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/schema-manage">Manage schema</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Pulsar Functions</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/functions-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/functions-worker">Setup: Pulsar Functions Worker</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/functions-runtime">Setup: Configure Functions runtime</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/functions-develop">How-to: Develop</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/functions-package">How-to: Package</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/functions-debug">How-to: Debug</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/functions-deploy">How-to: Deploy</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/functions-cli">Reference: CLI</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/window-functions-context">Window Functions: Context</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Pulsar IO</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/io-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/io-quickstart">Get started</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/io-use">Use</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/io-debug">Debug</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/io-connectors">Built-in connector</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/io-cdc">CDC connector</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/io-develop">Develop</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/io-cli">CLI</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Pulsar SQL</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/sql-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/sql-getting-started">Query data</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/sql-deployment-configurations">Configuration and deployment</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/sql-rest-api">REST APIs</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Tiered Storage</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/tiered-storage-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/tiered-storage-aws">AWS S3 offloader</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/tiered-storage-gcs">GCS offloader</a></li><li class="navListItem navListItemActive"><a class="navItem" href="/docs/en/2.7.2/tiered-storage-filesystem">Filesystem offloader</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/tiered-storage-azure">Azure BlobStore offloader</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Transactions</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/transactions">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/transactions-guarantee">Transactions Guarantee</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/transactions-api">Transactions API</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Kubernetes (Helm)</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/helm-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/helm-prepare">Prepare</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/helm-install">Install</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/helm-deploy">Deployment</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/helm-upgrade">Upgrade</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/helm-tools">Required Tools</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Deployment</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/deploy-aws">Amazon Web Services</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/deploy-kubernetes">Kubernetes</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/deploy-bare-metal">Bare metal</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/deploy-bare-metal-multi-cluster">Bare metal multi-cluster</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/deploy-docker">Docker</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/deploy-monitoring">Monitor</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Administration</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/administration-zk-bk">ZooKeeper and BookKeeper</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/administration-geo">Geo-replication</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/administration-pulsar-manager">Pulsar Manager</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/administration-stats">Pulsar statistics</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/administration-load-balance">Load balance</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/administration-proxy">Pulsar proxy</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/administration-upgrade">Upgrade</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/administration-isolation">Pulsar isolation</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Security</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/security-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/security-tls-transport">Transport Encryption using TLS</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/security-tls-authentication">Authentication using TLS</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/security-tls-keystore">Using TLS with KeyStore configure</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/security-jwt">Authentication using JWT</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/security-athenz">Authentication using Athenz</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/security-kerberos">Authentication using Kerberos</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/security-oauth2">Authentication using OAuth 2.0 access tokens</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/security-authorization">Authorization and ACLs</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/security-encryption">End-to-End Encryption</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/security-extending">Extending</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/security-bouncy-castle">Bouncy Castle Providers</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Performance</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/performance-pulsar-perf">Pulsar Perf</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Client Libraries</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/client-libraries">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/client-libraries-java">Java</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/client-libraries-go">Go</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/client-libraries-python">Python</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/client-libraries-cpp">C++</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/client-libraries-node">Node.js</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/client-libraries-websocket">WebSocket</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/client-libraries-dotnet">C#</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Admin API</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/admin-api-overview">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/admin-api-clusters">Clusters</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/admin-api-tenants">Tenants</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/admin-api-brokers">Brokers</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/admin-api-namespaces">Namespaces</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/admin-api-permissions">Permissions</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/admin-api-topics">Topics</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/admin-api-functions">Functions</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Adaptors</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/adaptors-kafka">Kafka client wrapper</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/adaptors-spark">Apache Spark</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/adaptors-storm">Apache Storm</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Cookbooks</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/cookbooks-compaction">Topic compaction</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/cookbooks-deduplication">Message deduplication</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/cookbooks-non-persistent">Non-persistent messaging</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/cookbooks-retention-expiry">Message retention and expiry</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/cookbooks-encryption">Encryption</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/cookbooks-message-queue">Message queue</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/cookbooks-bookkeepermetadata">BookKeeper Ledger Metadata</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Development</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/develop-tools">Simulation tools</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/developing-binary-protocol">Binary protocol</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/develop-schema">Custom schema storage</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/develop-load-manager">Modular load manager</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/develop-cpp">Building Pulsar C++ client</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Reference</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/reference-terminology">Terminology</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/reference-cli-tools">Pulsar CLI tools</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/reference-configuration">Pulsar configuration</a></li><li class="navListItem"><a class="navItem" href="/docs/en/2.7.2/reference-metrics">Pulsar Metrics</a></li></ul></div></div></section></div><script>
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
var links = coll[i].nextElementSibling.getElementsByTagName('*');
if (checkActiveCategory){
for (var j = 0; j < links.length; j++) {
if (links[j].classList.contains('navListItemActive')){
coll[i].nextElementSibling.classList.toggle('hide');
coll[i].childNodes[1].classList.toggle('rotate');
checkActiveCategory = false;
break;
}
}
}
coll[i].addEventListener('click', function() {
var arrow = this.childNodes[1];
arrow.classList.toggle('rotate');
var content = this.nextElementSibling;
content.classList.toggle('hide');
});
}
document.addEventListener('DOMContentLoaded', function() {
createToggler('#navToggler', '#docsNav', 'docsSliderActive');
createToggler('#tocToggler', 'body', 'tocActive');
var headings = document.querySelector('.toc-headings');
headings && headings.addEventListener('click', function(event) {
var el = event.target;
while(el !== headings){
if (el.tagName === 'A') {
document.body.classList.remove('tocActive');
break;
} else{
el = el.parentNode;
}
}
}, false);
function createToggler(togglerSelector, targetSelector, className) {
var toggler = document.querySelector(togglerSelector);
var target = document.querySelector(targetSelector);
if (!toggler) {
return;
}
toggler.onclick = function(event) {
event.preventDefault();
target.classList.toggle(className);
};
}
});
</script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/pulsar/edit/master/site2/docs/tiered-storage-filesystem.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Use filesystem offloader with Pulsar</h1></header><article><div><span><p>This chapter guides you through every step of installing and configuring the filesystem offloader and using it with Pulsar.</p>
<h2><a class="anchor" aria-hidden="true" id="installation"></a><a href="#installation" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Installation</h2>
<p>This section describes how to install the filesystem offloader.</p>
<h3><a class="anchor" aria-hidden="true" id="prerequisite"></a><a href="#prerequisite" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Prerequisite</h3>
<ul>
<li>Pulsar: 2.4.2 or higher versions</li>
</ul>
<h3><a class="anchor" aria-hidden="true" id="step"></a><a href="#step" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Step</h3>
<p>This example uses Pulsar 2.5.1.</p>
<ol>
<li><p>Download the Pulsar tarball using one of the following ways:</p>
<ul>
<li><p>Download the Pulsar tarball from the <a href="https://archive.apache.org/dist/pulsar/pulsar-2.5.1/apache-pulsar-2.5.1-bin.tar.gz">Apache mirror</a></p></li>
<li><p>Download the Pulsar tarball from the Pulsar <a href="https://pulsar.apache.org/download">download page</a></p></li>
<li><p>Use the <a href="https://www.gnu.org/software/wget">wget</a> command to dowload the Pulsar tarball.</p>
<pre><code class="hljs css language-shell">wget https://archive.apache.org/dist/pulsar/pulsar-2.5.1/apache-pulsar-2.5.1-bin.tar.gz
</code></pre></li>
</ul></li>
<li><p>Download and untar the Pulsar offloaders package.</p>
<pre><code class="hljs css language-bash">wget https://downloads.apache.org/pulsar/pulsar-2.5.1/apache-pulsar-offloaders-2.5.1-bin.tar.gz
tar xvfz apache-pulsar-offloaders-2.5.1-bin.tar.gz
</code></pre>
<blockquote>
<p><strong>Note</strong></p>
<ul>
<li><p>If you run Pulsar in a bare metal cluster, ensure that the <code>offloaders</code> tarball is unzipped in every broker's Pulsar directory.</p></li>
<li><p>If you run Pulsar in Docker or deploying Pulsar using a Docker image (such as K8S and DCOS), you can use the <code>apachepulsar/pulsar-all</code> image. The <code>apachepulsar/pulsar-all</code> image has already bundled tiered storage offloaders.</p></li>
</ul>
</blockquote></li>
<li><p>Copy the Pulsar offloaders as <code>offloaders</code> in the Pulsar directory.</p>
<pre><code class="hljs">mv apache-pulsar-offloaders<span class="hljs-number">-2.5</span><span class="hljs-number">.1</span>/offloaders apache-pulsar<span class="hljs-number">-2.5</span><span class="hljs-number">.1</span>/offloaders
ls offloaders
</code></pre>
<p><strong>Output</strong></p>
<pre><code class="hljs"><span class="hljs-selector-tag">tiered-storage-file-system-2</span><span class="hljs-selector-class">.5</span><span class="hljs-selector-class">.1</span><span class="hljs-selector-class">.nar</span>
<span class="hljs-selector-tag">tiered-storage-jcloud-2</span><span class="hljs-selector-class">.5</span><span class="hljs-selector-class">.1</span><span class="hljs-selector-class">.nar</span>
</code></pre>
<blockquote>
<p><strong>Note</strong></p>
<ul>
<li><p>If you run Pulsar in a bare metal cluster, ensure that <code>offloaders</code> tarball is unzipped in every broker's Pulsar directory.</p></li>
<li><p>If you run Pulsar in Docker or deploying Pulsar using a Docker image (such as K8s and DCOS), you can use the <code>apachepulsar/pulsar-all</code> image. The <code>apachepulsar/pulsar-all</code> image has already bundled tiered storage offloaders.</p></li>
</ul>
</blockquote></li>
</ol>
<h2><a class="anchor" aria-hidden="true" id="configuration"></a><a href="#configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Configuration</h2>
<blockquote>
<p><strong>Note</strong></p>
<p>Before offloading data from BookKeeper to filesystem, you need to configure some properties of the filesystem offloader driver.</p>
</blockquote>
<p>Besides, you can also configure the filesystem offloader to run it automatically or trigger it manually.</p>
<h3><a class="anchor" aria-hidden="true" id="configure-filesystem-offloader-driver"></a><a href="#configure-filesystem-offloader-driver" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Configure filesystem offloader driver</h3>
<p>You can configure the filesystem offloader driver in the <code>broker.conf</code> or <code>standalone.conf</code> configuration file.</p>
<div class="tabs"><div class="nav-tabs"><div id="tab-group-7986-tab-7987" class="nav-link active" data-group="group_7986" data-tab="tab-group-7986-content-7987">HDFS</div><div id="tab-group-7986-tab-7988" class="nav-link" data-group="group_7986" data-tab="tab-group-7986-content-7988">NFS</div></div><div class="tab-content"><div id="tab-group-7986-content-7987" class="tab-pane active" data-group="group_7986" tabindex="-1"><div><span><ul>
<li><p><strong>Required</strong> configurations are as below.</p>
<table>
<thead>
<tr><th>Parameter</th><th>Description</th><th>Example value</th></tr>
</thead>
<tbody>
<tr><td><code>managedLedgerOffloadDriver</code></td><td>Offloader driver name, which is case-insensitive.</td><td>filesystem</td></tr>
<tr><td><code>fileSystemURI</code></td><td>Connection address, which is the URI to access the default Hadoop distributed file system.</td><td><a href="hdfs://127.0.0.1:9000">hdfs://127.0.0.1:9000</a></td></tr>
<tr><td><code>offloadersDirectory</code></td><td>Offloader directory</td><td>offloaders</td></tr>
<tr><td><code>fileSystemProfilePath</code></td><td>Hadoop profile path. The configuration file is stored in the Hadoop profile path. It contains various settings for Hadoop performance tuning.</td><td>../conf/filesystem_offload_core_site.xml</td></tr>
</tbody>
</table>
</li>
<li><p><strong>Optional</strong> configurations are as below.</p>
<table>
<thead>
<tr><th>Parameter</th><th>Description</th><th>Example value</th></tr>
</thead>
<tbody>
<tr><td><code>managedLedgerMinLedgerRolloverTimeMinutes</code></td><td>Minimum time between ledger rollover for a topic. <br><br><strong>Note</strong>: it is not recommended to set this parameter in the production environment.</td><td>2</td></tr>
<tr><td><code>managedLedgerMaxEntriesPerLedger</code></td><td>Maximum number of entries to append to a ledger before triggering a rollover.<br><br><strong>Note</strong>: it is not recommended to set this parameter in the production environment.</td><td>5000</td></tr>
</tbody>
</table>
</li>
</ul>
</span></div></div><div id="tab-group-7986-content-7988" class="tab-pane" data-group="group_7986" tabindex="-1"><div><span><ul>
<li><p><strong>Required</strong> configurations are as below.</p>
<table>
<thead>
<tr><th>Parameter</th><th>Description</th><th>Example value</th></tr>
</thead>
<tbody>
<tr><td><code>managedLedgerOffloadDriver</code></td><td>Offloader driver name, which is case-insensitive.</td><td>filesystem</td></tr>
<tr><td><code>offloadersDirectory</code></td><td>Offloader directory</td><td>offloaders</td></tr>
<tr><td><code>fileSystemProfilePath</code></td><td>NFS profile path. The configuration file is stored in the NFS profile path. It contains various settings for performance tuning.</td><td>../conf/filesystem_offload_core_site.xml</td></tr>
</tbody>
</table>
</li>
<li><p><strong>Optional</strong> configurations are as below.</p>
<table>
<thead>
<tr><th>Parameter</th><th>Description</th><th>Example value</th></tr>
</thead>
<tbody>
<tr><td><code>managedLedgerMinLedgerRolloverTimeMinutes</code></td><td>Minimum time between ledger rollover for a topic. <br><br><strong>Note</strong>: it is not recommended to set this parameter in the production environment.</td><td>2</td></tr>
<tr><td><code>managedLedgerMaxEntriesPerLedger</code></td><td>Maximum number of entries to append to a ledger before triggering a rollover.<br><br><strong>Note</strong>: it is not recommended to set this parameter in the production environment.</td><td>5000</td></tr>
</tbody>
</table>
</li>
</ul>
</span></div></div></div></div>
<h3><a class="anchor" aria-hidden="true" id="run-filesystem-offloader-automatically"></a><a href="#run-filesystem-offloader-automatically" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Run filesystem offloader automatically</h3>
<p>You can configure the namespace policy to offload data automatically once a threshold is reached. The threshold is based on the size of data that a topic has stored on a Pulsar cluster. Once the topic storage reaches the threshold, an offload operation is triggered automatically.</p>
<table>
<thead>
<tr><th>Threshold value</th><th>Action</th></tr>
</thead>
<tbody>
<tr><td>&gt; 0</td><td>It triggers the offloading operation if the topic storage reaches its threshold.</td></tr>
<tr><td>= 0</td><td>It causes a broker to offload data as soon as possible.</td></tr>
<tr><td>&lt; 0</td><td>It disables automatic offloading operation.</td></tr>
</tbody>
</table>
<p>Automatic offload runs when a new segment is added to a topic log. If you set the threshold on a namespace, but few messages are being produced to the topic, the filesystem offloader does not work until the current segment is full.</p>
<p>You can configure the threshold using CLI tools, such as pulsar-admin.</p>
<h4><a class="anchor" aria-hidden="true" id="example"></a><a href="#example" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Example</h4>
<p>This example sets the filesystem offloader threshold to 10 MB using pulsar-admin.</p>
<pre><code class="hljs css language-bash">pulsar-admin namespaces <span class="hljs-built_in">set</span>-offload-threshold --size 10M my-tenant/my-namespace
</code></pre>
<blockquote>
<p><strong>Tip</strong></p>
<p>For more information about the <code>pulsar-admin namespaces set-offload-threshold options</code> command, including flags, descriptions, default values, and shorthands, see <a href="/docs/en/2.7.2/reference-pulsar-admin#set-offload-threshold">here</a>.</p>
</blockquote>
<h3><a class="anchor" aria-hidden="true" id="run-filesystem-offloader-manually"></a><a href="#run-filesystem-offloader-manually" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Run filesystem offloader manually</h3>
<p>For individual topics, you can trigger the filesystem offloader manually using one of the following methods:</p>
<ul>
<li><p>Use the REST endpoint.</p></li>
<li><p>Use CLI tools (such as pulsar-admin).</p></li>
</ul>
<p>To manually trigger the filesystem offloader via CLI tools, you need to specify the maximum amount of data (threshold) that should be retained on a Pulsar cluster for a topic. If the size of the topic data on the Pulsar cluster exceeds this threshold, segments from the topic are offloaded to the filesystem until the threshold is no longer exceeded. Older segments are offloaded first.</p>
<h4><a class="anchor" aria-hidden="true" id="example-1"></a><a href="#example-1" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Example</h4>
<ul>
<li><p>This example manually run the filesystem offloader using pulsar-admin.</p>
<pre><code class="hljs css language-bash">pulsar-admin topics offload --size-threshold 10M persistent://my-tenant/my-namespace/topic1
</code></pre>
<p><strong>Output</strong></p>
<pre><code class="hljs css language-bash">Offload triggered <span class="hljs-keyword">for</span> persistent://my-tenant/my-namespace/topic1 <span class="hljs-keyword">for</span> messages before 2:0:-1
</code></pre>
<blockquote>
<p><strong>Tip</strong></p>
<p>For more information about the <code>pulsar-admin topics offload options</code> command, including flags, descriptions, default values, and shorthands, see <a href="/docs/en/2.7.2/reference-pulsar-admin#offload">here</a>.</p>
</blockquote></li>
<li><p>This example checks filesystem offloader status using pulsar-admin.</p>
<pre><code class="hljs css language-bash">pulsar-admin topics offload-status persistent://my-tenant/my-namespace/topic1
</code></pre>
<p><strong>Output</strong></p>
<pre><code class="hljs css language-bash">Offload is currently running
</code></pre>
<p>To wait for the filesystem to complete the job, add the <code>-w</code> flag.</p>
<pre><code class="hljs css language-bash">pulsar-admin topics offload-status -w persistent://my-tenant/my-namespace/topic1
</code></pre>
<p><strong>Output</strong></p>
<pre><code class="hljs"><span class="hljs-attribute">Offload was a success</span>
</code></pre>
<p>If there is an error in the offloading operation, the error is propagated to the <code>pulsar-admin topics offload-status</code> command.</p>
<pre><code class="hljs css language-bash">pulsar-admin topics offload-status persistent://my-tenant/my-namespace/topic1
</code></pre>
<p><strong>Output</strong></p>
<pre><code class="hljs"><span class="hljs-builtin-name">Error</span> <span class="hljs-keyword">in</span> offload
<span class="hljs-literal">null</span>
Reason: <span class="hljs-builtin-name">Error</span> offloading: org.apache.bookkeeper.mledger.ManagedLedgerException: java.util.concurrent.CompletionException: com.amazonaws.services.s3.model.AmazonS3Exception: Anonymous<span class="hljs-built_in"> users </span>cannot initiate multipart uploads. Please authenticate. (Service: Amazon S3; Status Code: 403; <span class="hljs-builtin-name">Error</span> Code: AccessDenied; Request ID: 798758DE3F1776DF; S3 Extended Request ID: dhBFz/lZm1oiG/oBEepeNlhrtsDlzoOhocuYMpKihQGXe6EG8puRGOkK6UwqzVrMXTWBxxHcS+g=), S3 Extended Request ID: dhBFz/lZm1oiG/oBEepeNlhrtsDlzoOhocuYMpKihQGXe6EG8puRGOkK6UwqzVrMXTWBxxHcS+g=
</code></pre>
<blockquote>
<p><strong>Tip</strong></p>
<p>For more information about the <code>pulsar-admin topics offload-status options</code> command, including flags, descriptions, default values, and shorthands, see <a href="/docs/en/2.7.2/reference-pulsar-admin#offload-status">here</a>.</p>
</blockquote></li>
</ul>
<h2><a class="anchor" aria-hidden="true" id="tutorial"></a><a href="#tutorial" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Tutorial</h2>
<p>This section provides step-by-step instructions on how to use the filesystem offloader to move data from Pulsar to Hadoop Distributed File System (HDFS) or Network File system (NFS).</p>
<div class="tabs"><div class="nav-tabs"><div id="tab-group-7989-tab-7990" class="nav-link active" data-group="group_7989" data-tab="tab-group-7989-content-7990">HDFS</div><div id="tab-group-7989-tab-7991" class="nav-link" data-group="group_7989" data-tab="tab-group-7989-content-7991">NFS</div></div><div class="tab-content"><div id="tab-group-7989-content-7990" class="tab-pane active" data-group="group_7989" tabindex="-1"><div><span><p>To move data from Pulsar to HDFS, follow these steps.</p>
<h3><a class="anchor" aria-hidden="true" id="step-1-prepare-the-hdfs-environment"></a><a href="#step-1-prepare-the-hdfs-environment" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Step 1: Prepare the HDFS environment</h3>
<p>This tutorial sets up a Hadoop single node cluster and uses Hadoop 3.2.1.</p>
<blockquote>
<p><strong>Tip</strong></p>
<p>For details about how to set up a Hadoop single node cluster, see <a href="https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/SingleCluster.html">here</a>.</p>
</blockquote>
<ol>
<li><p>Download and uncompress Hadoop 3.2.1.</p>
<pre><code class="hljs">wget <span class="hljs-symbol">https:</span>/<span class="hljs-regexp">/mirrors.bfsu.edu.cn/apache</span><span class="hljs-regexp">/hadoop/common</span><span class="hljs-regexp">/hadoop-3.2.1/hadoop</span><span class="hljs-number">-3.2</span>.<span class="hljs-number">1</span>.tar.gz <br /><br />tar -zxvf hadoop<span class="hljs-number">-3.2</span>.<span class="hljs-number">1</span>.tar.gz -C <span class="hljs-variable">$HADOOP_HOME</span><br /></code></pre></li>
<li><p>Configure Hadoop.</p>
<pre><code class="hljs"><span class="hljs-meta"># $HADOOP_HOME/etc/hadoop/core-site.xml</span><br /><span class="hljs-params">&lt;configuration&gt;</span><br /> <span class="hljs-params">&lt;property&gt;</span><br /> <span class="hljs-params">&lt;name&gt;</span>fs.defaultFS<span class="hljs-params">&lt;/name&gt;</span><br /> <span class="hljs-params">&lt;value&gt;</span>hdfs:<span class="hljs-comment">//localhost:9000&lt;/value&gt;</span><br /> <span class="hljs-params">&lt;/property&gt;</span><br /><span class="hljs-params">&lt;/configuration&gt;</span><br /><br /><span class="hljs-meta"># $HADOOP_HOME/etc/hadoop/hdfs-site.xml</span><br /><span class="hljs-params">&lt;configuration&gt;</span><br /> <span class="hljs-params">&lt;property&gt;</span><br /> <span class="hljs-params">&lt;name&gt;</span>dfs.replication<span class="hljs-params">&lt;/name&gt;</span><br /> <span class="hljs-params">&lt;value&gt;</span><span class="hljs-number">1</span><span class="hljs-params">&lt;/value&gt;</span><br /> <span class="hljs-params">&lt;/property&gt;</span><br /><span class="hljs-params">&lt;/configuration&gt;</span><br /></code></pre></li>
<li><p>Set passphraseless ssh.</p>
<pre><code class="hljs"><span class="hljs-comment"># Now check that you can ssh to the localhost without a passphrase:</span><br /><span class="hljs-variable">$ </span>ssh localhost<br /><span class="hljs-comment"># If you cannot ssh to localhost without a passphrase, execute the following commands</span><br /><span class="hljs-variable">$ </span>ssh-keygen -t rsa -P <span class="hljs-string">''</span> -f ~<span class="hljs-regexp">/.ssh/id</span>_rsa<br /><span class="hljs-variable">$ </span>cat ~<span class="hljs-regexp">/.ssh/id</span>_rsa.pub &gt;&gt; ~<span class="hljs-regexp">/.ssh/authorized</span>_keys<br /><span class="hljs-variable">$ </span>chmod 0600 ~<span class="hljs-regexp">/.ssh/authorized</span>_keys<br /></code></pre></li>
<li><p>Start HDFS.</p>
<pre><code class="hljs"><span class="hljs-comment"># don't execute this command repeatedly, repeat execute will cauld the clusterId of the datanode is not consistent with namenode</span><br /><span class="hljs-variable">$HADOOP_HOME</span><span class="hljs-regexp">/bin/</span>hadoop namenode -format<br /><span class="hljs-variable">$HADOOP_HOME</span><span class="hljs-regexp">/sbin/</span>start-dfs.sh<br /></code></pre></li>
<li><p>Navigate to the <a href="http://localhost:9870/">HDFS website</a>.</p>
<p>You can see the <strong>Overview</strong> page.</p>
<p><img src="/docs/assets/FileSystem-1.png" alt=""></p>
<ol>
<li><p>At the top navigation bar, click <strong>Datanodes</strong> to check DataNode information.</p>
<p><img src="/docs/assets/FileSystem-2.png" alt=""></p></li>
<li><p>Click <strong>HTTP Address</strong> to get more detailed information about localhost:9866.</p>
<p>As can be seen below, the size of <strong>Capacity Used</strong> is 4 KB, which is the initial value.</p>
<p><img src="/docs/assets/FileSystem-3.png" alt=""></p></li>
</ol></li>
</ol>
<h3><a class="anchor" aria-hidden="true" id="step-2-install-the-filesystem-offloader"></a><a href="#step-2-install-the-filesystem-offloader" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Step 2: Install the filesystem offloader</h3>
<p>For details, see <a href="#installation">installation</a>.</p>
<h3><a class="anchor" aria-hidden="true" id="step-3-configure-the-filesystem-offloader"></a><a href="#step-3-configure-the-filesystem-offloader" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Step 3: Configure the filesystem offloader</h3>
<p>As indicated in the <a href="#configuration">configuration</a> section, you need to configure some properties for the filesystem offloader driver before using it. This tutorial assumes that you have configured the filesystem offloader driver as below and run Pulsar in <strong>standalone</strong> mode.</p>
<p>Set the following configurations in the <code>conf/standalone.conf</code> file.</p>
<pre><code class="hljs css language-conf"><span class="hljs-attr">managedLedgerOffloadDriver</span>=filesystem<br /><span class="hljs-attr">fileSystemURI</span>=hdfs://<span class="hljs-number">127.0</span>.<span class="hljs-number">0.1</span>:<span class="hljs-number">9000</span><br /><span class="hljs-attr">fileSystemProfilePath</span>=../conf/filesystem_<span class="hljs-literal">off</span>load_core_site.xml<br /></code></pre>
<blockquote>
<p><strong>Note</strong></p>
<p>For testing purposes, you can set the following two configurations to speed up ledger rollover, but it is not recommended that you set them in the production environment.</p>
</blockquote>
<pre><code class="hljs"><span class="hljs-attr">managedLedgerMinLedgerRolloverTimeMinutes</span>=<span class="hljs-number">1</span><br /><span class="hljs-attr">managedLedgerMaxEntriesPerLedger</span>=<span class="hljs-number">100</span><br /></code></pre>
</span></div></div><div id="tab-group-7989-content-7991" class="tab-pane" data-group="group_7989" tabindex="-1"><div><span><blockquote>
<p><strong>Note</strong></p>
<p>In this section, it is assumed that you have enabled NFS service and set the shared path of your NFS service. In this section, <code>/Users/test</code> is used as the shared path of NFS service.</p>
</blockquote>
<p>To offload data to NFS, follow these steps.</p>
<h3><a class="anchor" aria-hidden="true" id="step-1-install-the-filesystem-offloader"></a><a href="#step-1-install-the-filesystem-offloader" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Step 1: Install the filesystem offloader</h3>
<p>For details, see <a href="#installation">installation</a>.</p>
<h3><a class="anchor" aria-hidden="true" id="step-2-mont-your-nfs-to-your-local-filesystem"></a><a href="#step-2-mont-your-nfs-to-your-local-filesystem" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Step 2: Mont your NFS to your local filesystem</h3>
<p>This example mounts mounts <em>/Users/pulsar_nfs</em> to <em>/Users/test</em>.</p>
<pre><code class="hljs">mount -e <span class="hljs-number">192.168</span>.0.<span class="hljs-number">103</span><span class="hljs-symbol">:/Users/test/Users/pulsar_nfs</span><br /></code></pre>
<h3><a class="anchor" aria-hidden="true" id="step-3-configure-the-filesystem-offloader-driver"></a><a href="#step-3-configure-the-filesystem-offloader-driver" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Step 3: Configure the filesystem offloader driver</h3>
<p>As indicated in the <a href="#configuration">configuration</a> section, you need to configure some properties for the filesystem offloader driver before using it. This tutorial assumes that you have configured the filesystem offloader driver as below and run Pulsar in <strong>standalone</strong> mode.</p>
<ol>
<li><p>Set the following configurations in the <code>conf/standalone.conf</code> file.</p>
<pre><code class="hljs css language-conf"><span class="hljs-attr">managedLedgerOffloadDriver</span>=filesystem<br /><span class="hljs-attr">fileSystemProfilePath</span>=../conf/filesystem_<span class="hljs-literal">off</span>load_core_site.xml<br /></code></pre></li>
<li><p>Modify the <em>filesystem_offload_core_site.xml</em> as follows.</p>
<pre><code class="hljs"><span class="hljs-tag">&lt;<span class="hljs-name">property</span>&gt;</span><br /> <span class="hljs-tag">&lt;<span class="hljs-name">name</span>&gt;</span>fs.defaultFS<span class="hljs-tag">&lt;/<span class="hljs-name">name</span>&gt;</span><br /> <span class="hljs-tag">&lt;<span class="hljs-name">value</span>&gt;</span>file:///<span class="hljs-tag">&lt;/<span class="hljs-name">value</span>&gt;</span><br /><span class="hljs-tag">&lt;/<span class="hljs-name">property</span>&gt;</span><br /><br /><span class="hljs-tag">&lt;<span class="hljs-name">property</span>&gt;</span><br /> <span class="hljs-tag">&lt;<span class="hljs-name">name</span>&gt;</span>hadoop.tmp.dir<span class="hljs-tag">&lt;/<span class="hljs-name">name</span>&gt;</span><br /> <span class="hljs-tag">&lt;<span class="hljs-name">value</span>&gt;</span>file:///Users/pulsar_nfs<span class="hljs-tag">&lt;/<span class="hljs-name">value</span>&gt;</span><br /><span class="hljs-tag">&lt;/<span class="hljs-name">property</span>&gt;</span><br /><br /><span class="hljs-tag">&lt;<span class="hljs-name">property</span>&gt;</span><br /> <span class="hljs-tag">&lt;<span class="hljs-name">name</span>&gt;</span>io.file.buffer.size<span class="hljs-tag">&lt;/<span class="hljs-name">name</span>&gt;</span><br /> <span class="hljs-tag">&lt;<span class="hljs-name">value</span>&gt;</span>4096<span class="hljs-tag">&lt;/<span class="hljs-name">value</span>&gt;</span><br /><span class="hljs-tag">&lt;/<span class="hljs-name">property</span>&gt;</span><br /><br /><span class="hljs-tag">&lt;<span class="hljs-name">property</span>&gt;</span><br /> <span class="hljs-tag">&lt;<span class="hljs-name">name</span>&gt;</span>io.seqfile.compress.blocksize<span class="hljs-tag">&lt;/<span class="hljs-name">name</span>&gt;</span><br /> <span class="hljs-tag">&lt;<span class="hljs-name">value</span>&gt;</span>1000000<span class="hljs-tag">&lt;/<span class="hljs-name">value</span>&gt;</span><br /><span class="hljs-tag">&lt;/<span class="hljs-name">property</span>&gt;</span><br /><span class="hljs-tag">&lt;<span class="hljs-name">property</span>&gt;</span><br /><br /> <span class="hljs-tag">&lt;<span class="hljs-name">name</span>&gt;</span>io.seqfile.compression.type<span class="hljs-tag">&lt;/<span class="hljs-name">name</span>&gt;</span><br /> <span class="hljs-tag">&lt;<span class="hljs-name">value</span>&gt;</span>BLOCK<span class="hljs-tag">&lt;/<span class="hljs-name">value</span>&gt;</span><br /><span class="hljs-tag">&lt;/<span class="hljs-name">property</span>&gt;</span><br /><br /><span class="hljs-tag">&lt;<span class="hljs-name">property</span>&gt;</span><br /> <span class="hljs-tag">&lt;<span class="hljs-name">name</span>&gt;</span>io.map.index.interval<span class="hljs-tag">&lt;/<span class="hljs-name">name</span>&gt;</span><br /> <span class="hljs-tag">&lt;<span class="hljs-name">value</span>&gt;</span>128<span class="hljs-tag">&lt;/<span class="hljs-name">value</span>&gt;</span><br /><span class="hljs-tag">&lt;/<span class="hljs-name">property</span>&gt;</span><br /></code></pre></li>
</ol>
</span></div></div></div></div>
<h3><a class="anchor" aria-hidden="true" id="step-4-offload-data-from-bookkeeper-to-filesystem"></a><a href="#step-4-offload-data-from-bookkeeper-to-filesystem" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Step 4: Offload data from BookKeeper to filesystem</h3>
<p>Execute the following commands in the repository where you download Pulsar tarball. For example, <code>~/path/to/apache-pulsar-2.5.1</code>.</p>
<ol>
<li><p>Start Pulsar standalone.</p>
<pre><code class="hljs">bin/pulsar standalone -a <span class="hljs-number">127.0</span><span class="hljs-number">.0</span><span class="hljs-number">.1</span>
</code></pre></li>
<li><p>To ensure the data generated is not deleted immediately, it is recommended to set the <a href="https://pulsar.apache.org/docs/en/next/cookbooks-retention-expiry/#retention-policies">retention policy</a>, which can be either a <strong>size</strong> limit or a <strong>time</strong> limit. The larger value you set for the retention policy, the longer the data can be retained.</p>
<pre><code class="hljs"><span class="hljs-string">bin/</span><span class="hljs-string">pulsar-admin </span><span class="hljs-string">namespaces </span><span class="hljs-built_in">set-retention</span> <span class="hljs-string">public/</span><span class="hljs-string">default </span><span class="hljs-built_in">--size</span> <span class="hljs-string">100M </span><span class="hljs-built_in">--time</span> <span class="hljs-string">2d
</span></code></pre>
<blockquote>
<p><strong>Tip</strong></p>
<p>For more information about the <code>pulsarctl namespaces set-retention options</code> command, including flags, descriptions, default values, and shorthands, see <a href="https://docs.streamnative.io/pulsarctl/v2.7.0.6/#-em-set-retention-em-">here</a>.</p>
</blockquote></li>
<li><p>Produce data using pulsar-client.</p>
<pre><code class="hljs">bin/pulsar-client produce -m <span class="hljs-string">"Hello FileSystem Offloader"</span> -n <span class="hljs-number">1000</span> <span class="hljs-meta">public</span>/<span class="hljs-meta">default</span>/<span class="hljs-built_in">fs</span>-<span class="hljs-keyword">test</span>
</code></pre></li>
<li><p>The offloading operation starts after a ledger rollover is triggered. To ensure offload data successfully, it is recommended that you wait until several ledger rollovers are triggered. In this case, you might need to wait for a second. You can check the ledger status using pulsarctl.</p>
<pre><code class="hljs">bin/pulsar-<span class="hljs-keyword">admin</span> topics stats-<span class="hljs-type">internal</span> <span class="hljs-built_in">public</span>/<span class="hljs-keyword">default</span>/fs-test
</code></pre>
<p><strong>Output</strong></p>
<p>The data of the ledger 696 is not offloaded.</p>
<pre><code class="hljs">{
<span class="hljs-attr">"version"</span>: <span class="hljs-number">1</span>,
<span class="hljs-attr">"creationDate"</span>: <span class="hljs-string">"2020-06-16T21:46:25.807+08:00"</span>,
<span class="hljs-attr">"modificationDate"</span>: <span class="hljs-string">"2020-06-16T21:46:25.821+08:00"</span>,
<span class="hljs-attr">"ledgers"</span>: [
{
<span class="hljs-attr">"ledgerId"</span>: <span class="hljs-number">696</span>,
<span class="hljs-attr">"isOffloaded"</span>: <span class="hljs-literal">false</span>
}
],
<span class="hljs-attr">"cursors"</span>: {}
}
</code></pre></li>
<li><p>Wait a second and send more messages to the topic.</p>
<pre><code class="hljs">bin/pulsar-client produce -m <span class="hljs-string">"Hello FileSystem Offloader"</span> -n <span class="hljs-number">1000</span> <span class="hljs-meta">public</span>/<span class="hljs-meta">default</span>/<span class="hljs-built_in">fs</span>-<span class="hljs-keyword">test</span>
</code></pre></li>
<li><p>Check the ledger status using pulsarctl.</p>
<pre><code class="hljs">bin/pulsar-<span class="hljs-keyword">admin</span> topics stats-<span class="hljs-type">internal</span> <span class="hljs-built_in">public</span>/<span class="hljs-keyword">default</span>/fs-test
</code></pre>
<p><strong>Output</strong></p>
<p>The ledger 696 is rollovered.</p>
<pre><code class="hljs">{
<span class="hljs-attr">"version"</span>: <span class="hljs-number">2</span>,
<span class="hljs-attr">"creationDate"</span>: <span class="hljs-string">"2020-06-16T21:46:25.807+08:00"</span>,
<span class="hljs-attr">"modificationDate"</span>: <span class="hljs-string">"2020-06-16T21:48:52.288+08:00"</span>,
<span class="hljs-attr">"ledgers"</span>: [
{
<span class="hljs-attr">"ledgerId"</span>: <span class="hljs-number">696</span>,
<span class="hljs-attr">"entries"</span>: <span class="hljs-number">1001</span>,
<span class="hljs-attr">"size"</span>: <span class="hljs-number">81695</span>,
<span class="hljs-attr">"isOffloaded"</span>: <span class="hljs-literal">false</span>
},
{
<span class="hljs-attr">"ledgerId"</span>: <span class="hljs-number">697</span>,
<span class="hljs-attr">"isOffloaded"</span>: <span class="hljs-literal">false</span>
}
],
<span class="hljs-attr">"cursors"</span>: {}
}
</code></pre></li>
<li><p>Trigger the offloading operation manually using pulsarctl.</p>
<pre><code class="hljs">bin/pulsar-admin topics offload -s <span class="hljs-number">0</span> <span class="hljs-meta">public</span>/<span class="hljs-meta">default</span>/<span class="hljs-built_in">fs</span>-<span class="hljs-keyword">test</span>
</code></pre>
<p><strong>Output</strong></p>
<p>Data in ledgers before the ledge 697 is offloaded.</p>
<pre><code class="hljs"><span class="hljs-comment"># offload info, the ledgers before 697 will be offloaded</span>
Offload triggered <span class="hljs-keyword">for</span> persistent:<span class="hljs-regexp">//</span>public<span class="hljs-regexp">/default/</span>fs-test3 <span class="hljs-keyword">for</span> messages before <span class="hljs-number">697</span>:<span class="hljs-number">0</span>:-<span class="hljs-number">1</span>
</code></pre></li>
<li><p>Check the ledger status using pulsarctl.</p>
<pre><code class="hljs">bin/pulsar-<span class="hljs-keyword">admin</span> topics stats-<span class="hljs-type">internal</span> <span class="hljs-built_in">public</span>/<span class="hljs-keyword">default</span>/fs-test
</code></pre>
<p><strong>Output</strong></p>
<p>The data of the ledger 696 is offloaded.</p>
<pre><code class="hljs">{
<span class="hljs-attr">"version"</span>: <span class="hljs-number">4</span>,
<span class="hljs-attr">"creationDate"</span>: <span class="hljs-string">"2020-06-16T21:46:25.807+08:00"</span>,
<span class="hljs-attr">"modificationDate"</span>: <span class="hljs-string">"2020-06-16T21:52:13.25+08:00"</span>,
<span class="hljs-attr">"ledgers"</span>: [
{
<span class="hljs-attr">"ledgerId"</span>: <span class="hljs-number">696</span>,
<span class="hljs-attr">"entries"</span>: <span class="hljs-number">1001</span>,
<span class="hljs-attr">"size"</span>: <span class="hljs-number">81695</span>,
<span class="hljs-attr">"isOffloaded"</span>: <span class="hljs-literal">true</span>
},
{
<span class="hljs-attr">"ledgerId"</span>: <span class="hljs-number">697</span>,
<span class="hljs-attr">"isOffloaded"</span>: <span class="hljs-literal">false</span>
}
],
<span class="hljs-attr">"cursors"</span>: {}
}
</code></pre>
<p>And the <strong>Capacity Used</strong> is changed from 4 KB to 116.46 KB.</p>
<p><img src="/docs/assets/FileSystem-8.png" alt=""></p></li>
</ol>
</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/en/2.7.2/tiered-storage-gcs"><span class="arrow-prev">← </span><span>GCS offloader</span></a><a class="docs-next button" href="/docs/en/2.7.2/tiered-storage-azure"><span class="function-name-prevnext">Azure BlobStore offloader</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#installation">Installation</a><ul class="toc-headings"><li><a href="#prerequisite">Prerequisite</a></li><li><a href="#step">Step</a></li></ul></li><li><a href="#configuration">Configuration</a><ul class="toc-headings"><li><a href="#configure-filesystem-offloader-driver">Configure filesystem offloader driver</a></li><li><a href="#run-filesystem-offloader-automatically">Run filesystem offloader automatically</a></li><li><a href="#run-filesystem-offloader-manually">Run filesystem offloader manually</a></li></ul></li><li><a href="#tutorial">Tutorial</a><ul class="toc-headings"><li><a href="#step-1-prepare-the-hdfs-environment">Step 1: Prepare the HDFS environment</a></li><li><a href="#step-2-install-the-filesystem-offloader">Step 2: Install the filesystem offloader</a></li><li><a href="#step-3-configure-the-filesystem-offloader">Step 3: Configure the filesystem offloader</a></li><li><a href="#step-1-install-the-filesystem-offloader">Step 1: Install the filesystem offloader</a></li><li><a href="#step-2-mont-your-nfs-to-your-local-filesystem">Step 2: Mont your NFS to your local filesystem</a></li><li><a href="#step-3-configure-the-filesystem-offloader-driver">Step 3: Configure the filesystem offloader driver</a></li><li><a href="#step-4-offload-data-from-bookkeeper-to-filesystem">Step 4: Offload data from BookKeeper to filesystem</a></li></ul></li></ul></nav></div><footer class="nav-footer" id="footer"><section class="copyright">Copyright © 2022 The Apache Software Foundation. All Rights Reserved. Apache, Apache Pulsar and the Apache feather logo are trademarks of The Apache Software Foundation.</section><span><script>
const community = document.querySelector("a[href='#community']").parentNode;
const communityMenu =
'<li>' +
'<a id="community-menu" href="#">Community <span style="font-size: 0.75em">&nbsp;▼</span></a>' +
'<div id="community-dropdown" class="hide">' +
'<ul id="community-dropdown-items">' +
'<li><a href="/en/contact">Contact</a></li>' +
'<li><a href="/en/contributing">Contributing</a></li>' +
'<li><a href="/en/coding-guide">Coding guide</a></li>' +
'<li><a href="/en/events">Events</a></li>' +
'<li><a href="https://twitter.com/Apache_Pulsar" target="_blank">Twitter &#x2750</a></li>' +
'<li><a href="https://github.com/apache/pulsar/wiki" target="_blank">Wiki &#x2750</a></li>' +
'<li><a href="https://github.com/apache/pulsar/issues" target="_blank">Issue tracking &#x2750</a></li>' +
'<li><a href="https://pulsar-summit.org/" target="_blank">Pulsar Summit &#x2750</a></li>' +
'<li>&nbsp;</li>' +
'<li><a href="/en/resources">Resources</a></li>' +
'<li><a href="/en/team">Team</a></li>' +
'<li><a href="/en/powered-by">Powered By</a></li>' +
'</ul>' +
'</div>' +
'</li>';
community.innerHTML = communityMenu;
const communityMenuItem = document.getElementById("community-menu");
const communityDropDown = document.getElementById("community-dropdown");
communityMenuItem.addEventListener("click", function(event) {
event.preventDefault();
if (communityDropDown.className == 'hide') {
communityDropDown.className = 'visible';
} else {
communityDropDown.className = 'hide';
}
});
</script></span></footer></div><script>window.twttr=(function(d,s, id){var js,fjs=d.getElementsByTagName(s)[0],t=window.twttr||{};if(d.getElementById(id))return t;js=d.createElement(s);js.id=id;js.src='https://platform.twitter.com/widgets.js';fjs.parentNode.insertBefore(js, fjs);t._e = [];t.ready = function(f) {t._e.push(f);};return t;}(document, 'script', 'twitter-wjs'));</script></body></html>