| <!doctype html> |
| <html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-current plugin-docs plugin-id-default docs-doc-id-cookbooks-tiered-storage"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="generator" content="Docusaurus v2.4.0"> |
| <title data-rh="true">Tiered Storage | Apache Pulsar</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:image" content="https://pulsar.apache.org/img/pulsar-social-media-card.png"><meta data-rh="true" name="twitter:image" content="https://pulsar.apache.org/img/pulsar-social-media-card.png"><meta data-rh="true" property="og:url" content="https://pulsar.apache.org/docs/next/cookbooks-tiered-storage/"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="current"><meta data-rh="true" name="docusaurus_tag" content="docs-default-current"><meta data-rh="true" name="docsearch:version" content="current"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-current"><meta data-rh="true" property="og:title" content="Tiered Storage | Apache Pulsar"><meta data-rh="true" name="description" content="Pulsar's Tiered Storage feature allows older backlog data to be offloaded to long term storage, thereby freeing up space in BookKeeper and reducing storage costs. This cookbook walks you through using tiered storage in your Pulsar cluster."><meta data-rh="true" property="og:description" content="Pulsar's Tiered Storage feature allows older backlog data to be offloaded to long term storage, thereby freeing up space in BookKeeper and reducing storage costs. This cookbook walks you through using tiered storage in your Pulsar cluster."><link data-rh="true" rel="icon" href="/img/favicon.ico"><link data-rh="true" rel="alternate" href="https://pulsar.apache.org/docs/next/cookbooks-tiered-storage/" hreflang="en"><link data-rh="true" rel="alternate" href="https://pulsar.apache.org/docs/next/cookbooks-tiered-storage/" hreflang="x-default"><link data-rh="true" rel="canonical" href="https://pulsar.apache.org/docs/next/cookbooks-tiered-storage/"><link data-rh="true" rel="preconnect" href="https://WK2YL0SALL-dsn.algolia.net" crossorigin="anonymous"><link rel="alternate" type="application/rss+xml" href="/blog/rss.xml" title="Apache Pulsar RSS Feed"> |
| <link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Pulsar Atom Feed"> |
| |
| |
| |
| <link rel="search" type="application/opensearchdescription+xml" title="Apache Pulsar" href="/opensearch.xml"> |
| |
| |
| |
| |
| |
| <link rel="stylesheet" href="/css/katex-0.13.24.min.css" media="print" onload="this.media='all'"> |
| <script src="/js/sine-waves.min.js" async></script> |
| <script src="/js/matomo-agent.js"></script><link rel="stylesheet" href="/assets/css/styles.b0f65ef3.css"> |
| <link rel="preload" href="/assets/js/runtime~main.1d0ed2a7.js" as="script"> |
| <link rel="preload" href="/assets/js/main.e07a0c68.js" as="script"> |
| </head> |
| <body class="navigation-with-keyboard"> |
| <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus"> |
| <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" style="background-color:#282826;color:#fff" role="banner"><div class="content_knG7 announcementBarContent_xLdY"> |
| <a class="announcement-bar" href="https://registration.socio.events/e/pulsarvirtualsummiteurope2024" target="_blank"> |
| <div class="announcement-bar__content"> |
| <svg class="announcement-bar__icon"> |
| |
| <svg viewBox="0 0 33 32" fill="none" xmlns="http://www.w3.org/2000/svg"> |
| <path d="M6.5 19.6001H16.1L15.3 29.2001L26.5 12.4H17.06L18.1 2.80005L6.5 19.6001Z" stroke="#F7F7F7" stroke-width="1.5" stroke-linejoin="round"/> |
| </svg> |
| |
| </svg> |
| |
| <span> |
| Get your free pass for Pulsar Virtual Summit Europe 2024 on May 14, 2024 🗓️ |
| </span> |
| |
| <svg class="announcement-bar__icon"> |
| |
| <svg viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg"> |
| <rect width="20" height="20" transform="translate(6 6)" fill="white" fill-opacity="0.01"/> |
| <path d="M17.6667 10.1667L23.5 16.0001M23.5 16.0001L17.6667 21.8334M23.5 16.0001L8.5 16.0001" stroke="white" stroke-linecap="round" stroke-linejoin="round"/> |
| <rect x="0.5" y="0.5" width="31" height="31" rx="15.5" stroke="white"/> |
| </svg> |
| |
| </svg> |
| </div> |
| </a> |
| </div></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/logo-black.svg" alt="Apache Pulsar logo" class="themedImage_ToTc themedImage--light_HNdA" height="25" width="127"><img src="/img/logo-black.svg" alt="Apache Pulsar logo" class="themedImage_ToTc themedImage--dark_i4oU" height="25" width="127"></div><b class="navbar__title text--truncate"></b></a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Get Started</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/docs/3.2.x/concepts-overview/">Concepts</a></li><li><a class="dropdown__link" href="/docs/3.2.x/">Quickstart</a></li><li><a class="dropdown__link" href="/ecosystem/">Ecosystem</a></li></ul></div><a class="navbar__item navbar__link" href="/docs/next/">Docs</a><a class="navbar__item navbar__link" href="/features/">Features</a><a class="navbar__item navbar__link" href="/use-cases/">Use Cases</a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link community-dropdown">Community</a><ul class="dropdown__menu"><li><a class="dropdown__link scroll-link scroll-welcome" id="scroll-welcome" href="/community/">Welcome</a></li><li><a class="dropdown__link scroll-link scroll-discussions" id="scroll-discussions" href="/community/#section-discussions">Discussions</a></li><li><a class="dropdown__link scroll-link" id="scroll-governance" href="/community/#section-governance">Governance</a></li><li><a class="dropdown__link scroll-link" id="scroll-community" href="/community/#section-community">Meet the Community</a></li><li><a class="dropdown__link scroll-link" id="scroll-contribute" href="/community/#section-contribute">Contribute</a></li><li><a class="dropdown__link" href="/contribute/">Contribution Guide</a></li><li><a href="https://github.com/apache/pulsar/wiki" target="_blank" rel="noopener noreferrer" class="dropdown__link">Wiki</a></li><li><a href="https://github.com/apache/pulsar/issues" target="_blank" rel="noopener noreferrer" class="dropdown__link">Issue Tracking</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Learn</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/blog/">Blog</a></li><li><a class="dropdown__link" href="/books/">Books</a></li><li><a class="dropdown__link" href="/case-studies/">Case Studies</a></li><li><a class="dropdown__link" href="/articles/">Articles</a></li><li><a class="dropdown__link" href="/presentations/">Presentations</a></li><li><a class="dropdown__link" href="/events/">Events</a></li></ul></div></div><div class="navbar__items navbar__items--right"><a class="navbar__item navbar__link navbar_download_button" href="/download/">Download</a><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20" aria-hidden="true"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Tiered Storage</h1></header><p>Pulsar's <strong>Tiered Storage</strong> feature allows older backlog data to be offloaded to long term storage, thereby freeing up space in BookKeeper and reducing storage costs. This cookbook walks you through using tiered storage in your Pulsar cluster.</p><ul><li><p>Tiered storage uses <a href="https://jclouds.apache.org" target="_blank" rel="noopener noreferrer">Apache jclouds</a> to support <a href="https://aws.amazon.com/s3/" target="_blank" rel="noopener noreferrer">Amazon S3</a> and <a href="https://cloud.google.com/storage/" target="_blank" rel="noopener noreferrer">Google Cloud Storage</a>(GCS for short) |
| for long term storage. With Jclouds, it is easy to add support for more <a href="https://jclouds.apache.org/reference/providers/#blobstore-providers" target="_blank" rel="noopener noreferrer">cloud storage providers</a> in the future.</p></li><li><p>Tiered storage uses <a href="http://hadoop.apache.org/" target="_blank" rel="noopener noreferrer">Apache Hadoop</a> to support filesystem for long term storage. |
| With Hadoop, it is easy to add support for more filesystem in the future.</p></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="when-should-i-use-tiered-storage">When should I use Tiered Storage?<a href="#when-should-i-use-tiered-storage" class="hash-link" aria-label="Direct link to When should I use Tiered Storage?" title="Direct link to When should I use Tiered Storage?"></a></h2><p>Tiered storage should be used when you have a topic for which you want to keep a very long backlog for a long time. For example, if you have a topic containing user actions which you use to train your recommendation systems, you may want to keep that data for a long time, so that if you change your recommendation algorithm you can rerun it against your full user history.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="the-offloading-mechanism">The offloading mechanism<a href="#the-offloading-mechanism" class="hash-link" aria-label="Direct link to The offloading mechanism" title="Direct link to The offloading mechanism"></a></h2><p>A topic in Pulsar is backed by a log, known as a managed ledger. This log is composed of an ordered list of segments. Pulsar only every writes to the final segment of the log. All previous segments are sealed. The data within the segment is immutable. This is known as a segment oriented architecture.</p><p><img loading="lazy" alt="Tiered storage" src="/assets/images/pulsar-tiered-storage-72d8b53762992cfeaa58ae3b48dd2522.png" title="Tiered Storage" width="1110" height="697" class="img_ev3q"></p><p>The Tiered Storage offloading mechanism takes advantage of this segment oriented architecture. When offloading is requested, the segments of the log are copied, one-by-one, to tiered storage. All segments of the log, apart from the segment currently being written to can be offloaded.</p><p>On the broker, the administrator must configure the bucket and credentials for the cloud storage service. |
| The configured bucket must exist before attempting to offload. If it does not exist, the offload operation will fail.</p><p>Pulsar uses multi-part objects to upload the segment data. It is possible that a broker could crash while uploading the data. |
| We recommend you add a life cycle rule your bucket to expire incomplete multi-part upload after a day or two to avoid |
| getting charged for incomplete uploads.</p><p>When ledgers are offloaded to long term storage, you can still query data in the offloaded ledgers with Pulsar SQL.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="configuring-the-offload-driver">Configuring the offload driver<a href="#configuring-the-offload-driver" class="hash-link" aria-label="Direct link to Configuring the offload driver" title="Direct link to Configuring the offload driver"></a></h2><p>Offloading is configured in <code>broker.conf</code>.</p><p>At a minimum, the administrator must configure the driver, the bucket and the authenticating credentials. |
| There is also some other knobs to configure, like the bucket region, the max block size in backed storage, etc.</p><p>Currently we support driver of types:</p><ul><li><code>aws-s3</code>: <a href="https://aws.amazon.com/s3/" target="_blank" rel="noopener noreferrer">Simple Cloud Storage Service</a></li><li><code>google-cloud-storage</code>: <a href="https://cloud.google.com/storage/" target="_blank" rel="noopener noreferrer">Google Cloud Storage</a></li><li><code>filesystem</code>: <a href="http://hadoop.apache.org/" target="_blank" rel="noopener noreferrer">Filesystem Storage</a></li></ul><blockquote><p>Driver names are case-insensitive for driver's name. There is a third driver type, <code>s3</code>, which is identical to <code>aws-s3</code>, |
| though it requires that you specify an endpoint url using <code>s3ManagedLedgerOffloadServiceEndpoint</code>. This is useful if |
| using a S3 compatible data store, other than AWS.</p></blockquote><div class="language-conf codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-conf codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">managedLedgerOffloadDriver=aws-s3</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="aws-s3-driver-configuration">"aws-s3" Driver configuration<a href="#aws-s3-driver-configuration" class="hash-link" aria-label="Direct link to "aws-s3" Driver configuration" title="Direct link to "aws-s3" Driver configuration"></a></h3><h4 class="anchor anchorWithStickyNavbar_LWe7" id="bucket-and-region">Bucket and Region<a href="#bucket-and-region" class="hash-link" aria-label="Direct link to Bucket and Region" title="Direct link to Bucket and Region"></a></h4><p>Buckets are the basic containers that hold your data. |
| Everything that you store in Cloud Storage must be contained in a bucket. |
| You can use buckets to organize your data and control access to your data, |
| but unlike directories and folders, you cannot nest buckets.</p><div class="language-conf codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-conf codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">s3ManagedLedgerOffloadBucket=pulsar-topic-offload</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Bucket Region is the region where bucket located. Bucket Region is not a required |
| but a recommended configuration. If it is not configured, It will use the default region.</p><p>With AWS S3, the default region is <code>US East (N. Virginia)</code>. Page <a href="https://docs.aws.amazon.com/general/latest/gr/rande.html" target="_blank" rel="noopener noreferrer">AWS Regions and Endpoints</a> contains more information.</p><div class="language-conf codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-conf codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">s3ManagedLedgerOffloadRegion=eu-west-3</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="authentication-with-aws">Authentication with AWS<a href="#authentication-with-aws" class="hash-link" aria-label="Direct link to Authentication with AWS" title="Direct link to Authentication with AWS"></a></h4><p>To be able to access AWS S3, you need to authenticate with AWS S3. |
| Pulsar does not provide any direct means of configuring authentication for AWS S3, |
| but relies on the mechanisms supported by the <a href="https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html" target="_blank" rel="noopener noreferrer">DefaultAWSCredentialsProviderChain</a>.</p><p>Once you have created a set of credentials in the AWS IAM console, they can be configured in a number of ways.</p><ol><li>Using ec2 instance metadata credentials</li></ol><p>If you are on AWS instance with an instance profile that provides credentials, Pulsar will use these credentials |
| if no other mechanism is provided</p><ol start="2"><li>Set the environment variables <strong>AWS_ACCESS_KEY_ID</strong> and <strong>AWS_SECRET_ACCESS_KEY</strong> in <code>conf/pulsar_env.sh</code>.</li></ol><div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token builtin class-name" style="color:rgb(189, 147, 249)">export</span><span class="token plain"> </span><span class="token assign-left variable" style="color:rgb(189, 147, 249);font-style:italic">AWS_ACCESS_KEY_ID</span><span class="token operator">=</span><span class="token plain">ABC123456789</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token builtin class-name" style="color:rgb(189, 147, 249)">export</span><span class="token plain"> </span><span class="token assign-left variable" style="color:rgb(189, 147, 249);font-style:italic">AWS_SECRET_ACCESS_KEY</span><span class="token operator">=</span><span class="token plain">ded7db27a4558e2ea8bbf0bf37ae0e8521618f366c</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><blockquote><p>\"export\" is important so that the variables are made available in the environment of spawned processes.</p></blockquote><ol start="3"><li>Add the Java system properties <em>aws.accessKeyId</em> and <em>aws.secretKey</em> to <strong>PULSAR_EXTRA_OPTS</strong> in <code>conf/pulsar_env.sh</code>.</li></ol><div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token assign-left variable" style="color:rgb(189, 147, 249);font-style:italic">PULSAR_EXTRA_OPTS</span><span class="token operator">=</span><span class="token string" style="color:rgb(255, 121, 198)">"</span><span class="token string variable" style="color:rgb(189, 147, 249);font-style:italic">${PULSAR_EXTRA_OPTS}</span><span class="token string" style="color:rgb(255, 121, 198)"> </span><span class="token string variable" style="color:rgb(189, 147, 249);font-style:italic">${PULSAR_MEM}</span><span class="token string" style="color:rgb(255, 121, 198)"> </span><span class="token string variable" style="color:rgb(189, 147, 249);font-style:italic">${PULSAR_GC}</span><span class="token string" style="color:rgb(255, 121, 198)"> -Daws.accessKeyId=ABC123456789 -Daws.secretKey=ded7db27a4558e2ea8bbf0bf37ae0e8521618f366c -Dio.netty.leakDetectionLevel=disabled -Dio.netty.recycler.maxCapacityPerThread=4096"</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><ol start="4"><li>Set the access credentials in <code>~/.aws/credentials</code>.</li></ol><div class="language-conf codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-conf codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">[default]</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">aws_access_key_id=ABC123456789</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">aws_secret_access_key=ded7db27a4558e2ea8bbf0bf37ae0e8521618f366c</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><ol start="5"><li>Assuming an IAM role</li></ol><p>If you want to assume an IAM role, this can be done via specifying the following:</p><div class="language-conf codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-conf codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">s3ManagedLedgerOffloadRole=<aws role arn></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">s3ManagedLedgerOffloadRoleSessionName=pulsar-s3-offload</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This will use the <code>DefaultAWSCredentialsProviderChain</code> for assuming this role.</p><blockquote><p>The broker must be rebooted for credentials specified in pulsar_env to take effect.</p></blockquote><h4 class="anchor anchorWithStickyNavbar_LWe7" id="configuring-the-size-of-block-readwrite">Configuring the size of block read/write<a href="#configuring-the-size-of-block-readwrite" class="hash-link" aria-label="Direct link to Configuring the size of block read/write" title="Direct link to Configuring the size of block read/write"></a></h4><p>Pulsar also provides some knobs to configure the size of requests sent to AWS S3.</p><ul><li><code>s3ManagedLedgerOffloadMaxBlockSizeInBytes</code> configures the maximum size of |
| a "part" sent during a multipart upload. This cannot be smaller than 5MB. Default is 64MB.</li><li><code>s3ManagedLedgerOffloadReadBufferSizeInBytes</code> configures the block size for |
| each individual read when reading back data from AWS S3. Default is 1MB.</li></ul><p>In both cases, these should not be touched unless you know what you are doing.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="google-cloud-storage-driver-configuration">"google-cloud-storage" Driver configuration<a href="#google-cloud-storage-driver-configuration" class="hash-link" aria-label="Direct link to "google-cloud-storage" Driver configuration" title="Direct link to "google-cloud-storage" Driver configuration"></a></h3><p>Buckets are the basic containers that hold your data. Everything that you store in |
| Cloud Storage must be contained in a bucket. You can use buckets to organize your data and |
| control access to your data, but unlike directories and folders, you cannot nest buckets.</p><div class="language-conf codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-conf codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">gcsManagedLedgerOffloadBucket=pulsar-topic-offload</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Bucket Region is the region where bucket located. Bucket Region is not a required but |
| a recommended configuration. If it is not configured, It will use the default region.</p><p>Regarding GCS, buckets are default created in the <code>us multi-regional location</code>, |
| page <a href="https://cloud.google.com/storage/docs/bucket-locations" target="_blank" rel="noopener noreferrer">Bucket Locations</a> contains more information.</p><div class="language-conf codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-conf codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">gcsManagedLedgerOffloadRegion=europe-west3</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="authentication-with-gcs">Authentication with GCS<a href="#authentication-with-gcs" class="hash-link" aria-label="Direct link to Authentication with GCS" title="Direct link to Authentication with GCS"></a></h4><p>The administrator needs to configure <code>gcsManagedLedgerOffloadServiceAccountKeyFile</code> in <code>broker.conf</code> |
| for the broker to be able to access the GCS service. <code>gcsManagedLedgerOffloadServiceAccountKeyFile</code> is |
| a Json file, containing the GCS credentials of a service account. |
| <a href="https://support.google.com/googleapi/answer/6158849" target="_blank" rel="noopener noreferrer">Service Accounts section of this page</a> contains |
| more information of how to create this key file for authentication. More information about google cloud IAM |
| is available <a href="https://cloud.google.com/storage/docs/access-control/iam" target="_blank" rel="noopener noreferrer">here</a>.</p><p>To generate service account credentials or view the public credentials that you've already generated, follow the following steps:</p><ol><li>Open the <a href="https://console.developers.google.com/iam-admin/serviceaccounts" target="_blank" rel="noopener noreferrer">Service accounts page</a>.</li><li>Select a project or create a new one.</li><li>Click <strong>Create service account</strong>.</li><li>In the <strong>Create service account</strong> window, type a name for the service account, and select <strong>Furnish a new private key</strong>. If you want to <a href="https://developers.google.com/identity/protocols/OAuth2ServiceAccount#delegatingauthority" target="_blank" rel="noopener noreferrer">grant G Suite domain-wide authority</a> to the service account, also select <strong>Enable G Suite Domain-wide Delegation</strong>.</li><li>Click <strong>Create</strong>.</li></ol><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>Make ensure that the service account you create has permission to operate GCS, you need to assign <strong>Storage Admin</strong> permission to your service account in <a href="https://cloud.google.com/storage/docs/access-control/iam" target="_blank" rel="noopener noreferrer">here</a>.</p></div></div><div class="language-conf codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-conf codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">gcsManagedLedgerOffloadServiceAccountKeyFile="/Users/hello/Downloads/project-804d5e6a6f33.json"</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="configuring-the-size-of-block-readwrite-1">Configuring the size of block read/write<a href="#configuring-the-size-of-block-readwrite-1" class="hash-link" aria-label="Direct link to Configuring the size of block read/write" title="Direct link to Configuring the size of block read/write"></a></h4><p>Pulsar also provides some knobs to configure the size of requests sent to GCS.</p><ul><li><code>gcsManagedLedgerOffloadMaxBlockSizeInBytes</code> configures the maximum size of a "part" sent |
| during a multipart upload. This cannot be smaller than 5MB. Default is 64MB.</li><li><code>gcsManagedLedgerOffloadReadBufferSizeInBytes</code> configures the block size for each individual |
| read when reading back data from GCS. Default is 1MB.</li></ul><p>In both cases, these should not be touched unless you know what you are doing.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="filesystem-driver-configuration">"filesystem" Driver configuration<a href="#filesystem-driver-configuration" class="hash-link" aria-label="Direct link to "filesystem" Driver configuration" title="Direct link to "filesystem" Driver configuration"></a></h3><h4 class="anchor anchorWithStickyNavbar_LWe7" id="configure-connection-address">Configure connection address<a href="#configure-connection-address" class="hash-link" aria-label="Direct link to Configure connection address" title="Direct link to Configure connection address"></a></h4><p>You can configure the connection address in the <code>broker.conf</code> file.</p><div class="language-conf codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-conf codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">fileSystemURI="hdfs://127.0.0.1:9000"</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="configure-hadoop-profile-path">Configure Hadoop profile path<a href="#configure-hadoop-profile-path" class="hash-link" aria-label="Direct link to Configure Hadoop profile path" title="Direct link to Configure Hadoop profile path"></a></h4><p>The configuration file is stored in the Hadoop profile path. It contains various settings, such as base path, authentication, and so on.</p><div class="language-conf codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-conf codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">fileSystemProfilePath="../conf/filesystem_offload_core_site.xml"</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The model for storing topic data uses <code>org.apache.hadoop.io.MapFile</code>. You can use all of the configurations in <code>org.apache.hadoop.io.MapFile</code> for Hadoop.</p><p><strong>Example</strong></p><div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"><</span><span class="token tag" style="color:rgb(255, 121, 198)">property</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"><</span><span class="token tag" style="color:rgb(255, 121, 198)">name</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain">fs.defaultFS</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"></</span><span class="token tag" style="color:rgb(255, 121, 198)">name</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"><</span><span class="token tag" style="color:rgb(255, 121, 198)">value</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"></</span><span class="token tag" style="color:rgb(255, 121, 198)">value</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"></</span><span class="token tag" style="color:rgb(255, 121, 198)">property</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"><</span><span class="token tag" style="color:rgb(255, 121, 198)">property</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"><</span><span class="token tag" style="color:rgb(255, 121, 198)">name</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain">hadoop.tmp.dir</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"></</span><span class="token tag" style="color:rgb(255, 121, 198)">name</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"><</span><span class="token tag" style="color:rgb(255, 121, 198)">value</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain">pulsar</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"></</span><span class="token tag" style="color:rgb(255, 121, 198)">value</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"></</span><span class="token tag" style="color:rgb(255, 121, 198)">property</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"><</span><span class="token tag" style="color:rgb(255, 121, 198)">property</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"><</span><span class="token tag" style="color:rgb(255, 121, 198)">name</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain">io.file.buffer.size</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"></</span><span class="token tag" style="color:rgb(255, 121, 198)">name</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"><</span><span class="token tag" style="color:rgb(255, 121, 198)">value</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain">4096</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"></</span><span class="token tag" style="color:rgb(255, 121, 198)">value</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"></</span><span class="token tag" style="color:rgb(255, 121, 198)">property</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"><</span><span class="token tag" style="color:rgb(255, 121, 198)">property</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"><</span><span class="token tag" style="color:rgb(255, 121, 198)">name</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain">io.seqfile.compress.blocksize</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"></</span><span class="token tag" style="color:rgb(255, 121, 198)">name</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"><</span><span class="token tag" style="color:rgb(255, 121, 198)">value</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain">1000000</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"></</span><span class="token tag" style="color:rgb(255, 121, 198)">value</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"></</span><span class="token tag" style="color:rgb(255, 121, 198)">property</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"><</span><span class="token tag" style="color:rgb(255, 121, 198)">property</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"><</span><span class="token tag" style="color:rgb(255, 121, 198)">name</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain">io.seqfile.compression.type</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"></</span><span class="token tag" style="color:rgb(255, 121, 198)">name</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"><</span><span class="token tag" style="color:rgb(255, 121, 198)">value</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain">BLOCK</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"></</span><span class="token tag" style="color:rgb(255, 121, 198)">value</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"></</span><span class="token tag" style="color:rgb(255, 121, 198)">property</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"><</span><span class="token tag" style="color:rgb(255, 121, 198)">property</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"><</span><span class="token tag" style="color:rgb(255, 121, 198)">name</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain">io.map.index.interval</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"></</span><span class="token tag" style="color:rgb(255, 121, 198)">name</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"><</span><span class="token tag" style="color:rgb(255, 121, 198)">value</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain">128</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"></</span><span class="token tag" style="color:rgb(255, 121, 198)">value</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token tag punctuation" style="color:rgb(248, 248, 242)"></</span><span class="token tag" style="color:rgb(255, 121, 198)">property</span><span class="token tag punctuation" style="color:rgb(248, 248, 242)">></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>For more information about the configurations in <code>org.apache.hadoop.io.MapFile</code>, see <a href="http://hadoop.apache.org/" target="_blank" rel="noopener noreferrer">Filesystem Storage</a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="configuring-offload-to-run-automatically">Configuring offload to run automatically<a href="#configuring-offload-to-run-automatically" class="hash-link" aria-label="Direct link to Configuring offload to run automatically" title="Direct link to Configuring offload to run automatically"></a></h2><p>Namespace policies can be configured to offload data automatically once a threshold is reached. The threshold is based on the size of data that the topic has stored on the pulsar cluster. Once the topic reaches the threshold, an offload operation will be triggered. Setting a negative value to the threshold will disable automatic offloading. Setting the threshold to 0 will cause the broker to offload data as soon as it possiby can.</p><div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">bin/pulsar-admin namespaces set-offload-threshold --size 10M my-tenant/my-namespace</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><blockquote><p>Automatic offload runs when a new segment is added to a topic log. If you set the threshold on a namespace, but few messages are being produced to the topic, offload will not until the current segment is full.</p></blockquote><h2 class="anchor anchorWithStickyNavbar_LWe7" id="configuring-read-priority-for-offloaded-messages">Configuring read priority for offloaded messages<a href="#configuring-read-priority-for-offloaded-messages" class="hash-link" aria-label="Direct link to Configuring read priority for offloaded messages" title="Direct link to Configuring read priority for offloaded messages"></a></h2><p>By default, once messages were offloaded to long term storage, brokers will read them from long term storage, but messages still exists in bookkeeper for a period depends on the administrator's configuration. For |
| messages exists in both bookkeeper and long term storage, if they are preferred to read from bookkeeper, you can use command to change this configuration.</p><div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token comment" style="color:rgb(98, 114, 164)"># default value for -orp is tiered-storage-first</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">bin/pulsar-admin namespaces set-offload-policies my-tenant/my-namespace -orp bookkeeper-first</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">bin/pulsar-admin topics set-offload-policies my-tenant/my-namespace/topic1 -orp bookkeeper-first</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="triggering-offload-manually">Triggering offload manually<a href="#triggering-offload-manually" class="hash-link" aria-label="Direct link to Triggering offload manually" title="Direct link to Triggering offload manually"></a></h2><p>Offloading can manually triggered through a REST endpoint on the Pulsar broker. We provide a CLI which will call this rest endpoint for you.</p><p>When triggering offload, you must specify the maximum size, in bytes, of backlog which will be retained locally on the bookkeeper. The offload mechanism will offload segments from the start of the topic backlog until this condition is met.</p><div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">bin/pulsar-admin topics offload --size-threshold 10M my-tenant/my-namespace/topic1</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">Offload triggered </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">for</span><span class="token plain"> persistent://my-tenant/my-namespace/topic1 </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">for</span><span class="token plain"> messages before </span><span class="token number">2</span><span class="token plain">:0:-1</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The command to triggers an offload will not wait until the offload operation has completed. To check the status of the offload, use offload-status.</p><div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">bin/pulsar-admin topics offload-status my-tenant/my-namespace/topic1</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">Offload is currently running</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>To wait for offload to complete, add the -w flag.</p><div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">bin/pulsar-admin topics offload-status -w my-tenant/my-namespace/topic1</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">Offload was a success</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>If there is an error offloading, the error will be propagated to the offload-status command.</p><div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">bin/pulsar-admin topics offload-status persistent://public/default/topic1</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">Error </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">in</span><span class="token plain"> offload</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">null</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">Reason: Error offloading: org.apache.bookkeeper.mledger.ManagedLedgerException: java.util.concurrent.CompletionException: com.amazonaws.services.s3.model.AmazonS3Exception: Anonymous </span><span class="token function" style="color:rgb(80, 250, 123)">users</span><span class="token plain"> cannot initiate multipart uploads. Please authenticate. </span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token plain">Service: Amazon S3</span><span class="token punctuation" style="color:rgb(248, 248, 242)">;</span><span class="token plain"> Status Code: </span><span class="token number">403</span><span class="token punctuation" style="color:rgb(248, 248, 242)">;</span><span class="token plain"> Error Code: AccessDenied</span><span class="token punctuation" style="color:rgb(248, 248, 242)">;</span><span class="token plain"> Request ID: 798758DE3F1776DF</span><span class="token punctuation" style="color:rgb(248, 248, 242)">;</span><span class="token plain"> S3 Extended Request ID: dhBFz/lZm1oiG/oBEepeNlhrtsDlzoOhocuYMpKihQGXe6EG8puRGOkK6UwqzVrMXTWBxxHcS+g</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token plain">, S3 Extended Request ID: dhBFz/lZm1oiG/oBEepeNlhrtsDlzoOhocuYMpKihQGXe6EG8puRGOkK6UwqzVrMXTWBxxHcS+g</span><span class="token operator">=</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><footer class="theme-doc-footer docusaurus-mt-lg"><div class="theme-doc-footer-edit-meta-row row"><div class="col"><a href="https://github.com/apache/pulsar-site/edit/main/docs/cookbooks-tiered-storage.md" target="_blank" rel="noreferrer noopener" class="theme-edit-this-page"><svg fill="currentColor" height="20" width="20" viewBox="0 0 40 40" class="iconEdit_Z9Sw" aria-hidden="true"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div><div class="col lastUpdated_vwxv"></div></div></footer></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages navigation"></nav></div></div><div class="col col--3"><div class="tableOfContents_jeP5 thin-scrollbar theme-doc-toc-desktop"><div class="border"><div style="color:var(--ifm-toc-link-color)">Was this helpful?</div><div style="border-width:1px;padding:3px;display:flex"><div style="justify-content:center;display:flex;border-radius:99999px;width:2.5rem;height:2.5rem;cursor:pointer;background:;color:"><svg style="width:initial;height:initial" width="12" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M10.086 1.594A1 1 0 0 1 11 1a4 4 0 0 1 4 4v3h4.655a3 3 0 0 1 2.994 3.45l-1.38 9A3.002 3.002 0 0 1 18.275 23H4a3 3 0 0 1-3-3v-7a3 3 0 0 1 3-3h2.35l3.736-8.406ZM8 11.212l3.608-8.117A2 2 0 0 1 13 5v4a1 1 0 0 0 1 1h5.671a1 1 0 0 1 1 1.15l-1.38 9a1 1 0 0 1-1 .85H8v-9.788ZM6 21v-9H4a1 1 0 0 0-1 1v7a1 1 0 0 0 1 1h2Z" fill="currentColor"></path></svg></div><div style="justify-content:center;display:flex;border-radius:99999px;width:2.5rem;height:2.5rem;cursor:pointer;background:;color:"><svg style="width:initial;height:initial" width="12" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M20.563 3.316A1.31 1.31 0 0 0 19.687 3h-1.688v9h1.688a1.31 1.31 0 0 0 1.312-1.077V4.077a1.31 1.31 0 0 0-.436-.761ZM16 12.788l-3.608 8.117A1.999 1.999 0 0 1 11 19v-4a1 1 0 0 0-1-1H4.328a1.002 1.002 0 0 1-1-1.15l1.38-9a1 1 0 0 1 1-.85h10.291v9.788ZM19.661 1a3.31 3.31 0 0 1 3.329 2.866c.006.044.01.09.01.134v7c0 .045-.004.09-.01.134A3.31 3.31 0 0 1 19.661 14h-2.012l-3.736 8.406a1 1 0 0 1-.914.594 4 4 0 0 1-4-4v-3H4.344a3 3 0 0 1-2.994-3.45l1.38-9A3.002 3.002 0 0 1 5.724 1h13.937Z" fill="currentColor"></path></svg></div></div><div class="Actions_uugI"><a target="_blank" class="Action_iBHd" href="https://github.com/apache/pulsar/issues/new?assignees=&labels=doc-required&projects=&template=doc.yml&title=%5BDoc%5D+">💡 Suggest changes</a><a target="_blank" class="Action_iBHd" href="https://github.com/apache/pulsar/discussions/new?category=q-a">🛟 Get support</a></div></div><ul class="table-of-contents table-of-contents__left-border"><li><a href="#when-should-i-use-tiered-storage" class="table-of-contents__link toc-highlight">When should I use Tiered Storage?</a></li><li><a href="#the-offloading-mechanism" class="table-of-contents__link toc-highlight">The offloading mechanism</a></li><li><a href="#configuring-the-offload-driver" class="table-of-contents__link toc-highlight">Configuring the offload driver</a><ul><li><a href="#aws-s3-driver-configuration" class="table-of-contents__link toc-highlight">"aws-s3" Driver configuration</a></li><li><a href="#google-cloud-storage-driver-configuration" class="table-of-contents__link toc-highlight">"google-cloud-storage" Driver configuration</a></li><li><a href="#filesystem-driver-configuration" class="table-of-contents__link toc-highlight">"filesystem" Driver configuration</a></li></ul></li><li><a href="#configuring-offload-to-run-automatically" class="table-of-contents__link toc-highlight">Configuring offload to run automatically</a></li><li><a href="#configuring-read-priority-for-offloaded-messages" class="table-of-contents__link toc-highlight">Configuring read priority for offloaded messages</a></li><li><a href="#triggering-offload-manually" class="table-of-contents__link toc-highlight">Triggering offload manually</a></li></ul></div></div></div></div></main></div></div><footer class="footer"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title"></div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer" class="footer__link-item">Foundation<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://www.apache.org/events/current-event.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Events<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><div class="col footer__col"><div class="footer__title"></div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://www.apache.org/licenses/" target="_blank" rel="noopener noreferrer" class="footer__link-item">License<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://www.apache.org/foundation/thanks" target="_blank" rel="noopener noreferrer" class="footer__link-item">Thanks<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://www.apache.org/foundation/sponsorship" target="_blank" rel="noopener noreferrer" class="footer__link-item">Sponsorship<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><div class="col footer__col"><div class="footer__title"></div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/security/">Security</a></li><li class="footer__item"><a href="https://www.apache.org/foundation/policies/privacy.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Privacy<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a class="footer__link-item" href="/contact/">Contact</a></li></ul></div><div class="col footer__col"><div class="footer__title"></div><ul class="footer__items clean-list"><li class="footer__item"> |
| <div class="social-icons"> |
| <a target="_blank" href="https://communityinviter.com/apps/apache-pulsar/apache-pulsar" aria-label="Join the Apache Pulsar Slack workspace"> |
| <img alt="Slack logo" src="/img/slack-white.svg" width="26"> |
| </a> |
| <a target="_blank" href="https://github.com/apache/pulsar/" aria-label="View the Apache Pulsar project on GitHub"> |
| <img alt="GitHub logo" src="/img/github-white.svg" width="26"> |
| </a> |
| </div> |
| </li></ul></div></div><div class="footer__bottom text--center"><div class="margin-bottom--sm"><a class="footerLogoLink_BH7S" href="/"><img src="/img/pulsar-white.svg" alt="Pulsar Logo" class="themedImage_ToTc themedImage--light_HNdA footer__logo"><img src="/img/pulsar-white.svg" alt="Pulsar Logo" class="themedImage_ToTc themedImage--dark_i4oU footer__logo"></a></div><div class="footer__copyright"> |
| <div> |
| <img class="footer-apache-logo" src="/img/feather-logo-white.svg" alt="" width="20"> |
| The Apache Software Foundation |
| </div> |
| <p>Apache Pulsar is available under the Apache License, version 2.0. Apache Pulsar is an open-source, distributed messaging and streaming platform built for the cloud.</p> |
| <p>Copyright © 2024 The Apache Software Foundation. All Rights Reserved. Apache, Pulsar, Apache Pulsar, and the Apache feather logo are trademarks or registered trademarks of The Apache Software Foundation.</p> |
| </div></div></div></footer></div> |
| <script src="/assets/js/runtime~main.1d0ed2a7.js"></script> |
| <script src="/assets/js/main.e07a0c68.js"></script> |
| </body> |
| </html> |
