| <!DOCTYPE html> |
| <html lang="en"> |
| <head> |
| <meta charset="utf-8"> |
| <meta name="viewport" content="width=device-width,initial-scale=1.0"> |
| <title>Apache Cassandra | Apache Cassandra Documentation</title> |
| <link rel="stylesheet" href="../../assets/css/site.css"> |
| <meta name="description" content="The Apache Cassandra Community"> |
| <link rel="schema.dcterms" href="https://purl.org/dc/terms/"> |
| <meta name="dcterms.subject" content="_"> |
| <meta name="dcterms.identifier" content="master"> |
| <meta name="generator" content="Antora 2.3.4"> |
| <link rel="icon" href="../../assets/img/favicon.ico" type="image/x-icon"> |
| <script> |
| const script = document.createElement("script"); |
| const domain = window.location.hostname; |
| script.type = "text/javascript"; |
| script.src = "https://plausible.cassandra.apache.org/js/plausible.js"; |
| script.setAttribute("data-domain",domain); |
| script.setAttribute("defer",'true'); |
| script.setAttribute("async",'true'); |
| document.getElementsByTagName("head")[0].appendChild(script); |
| </script> </head> |
| <body class="single-post"> |
| <div class="container mx-auto relative"> |
| <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.6.0/jquery.min.js"></script> |
| <meta property="og:type" content="website" /> |
| <meta property="og:description" content="" /> |
| <meta property="og:url" content="/" /> |
| <meta property="og:site_name" content="Apache Cassandra" /> |
| |
| <header id="top-nav"> |
| <div class="inner relative"> |
| <div class="header-social-icons text-right"> |
| <a href="https://twitter.com/cassandra?lang=en" target="_blank" styles="margin-left: 20px;"><img src="../../assets/img/twitter-icon-circle-white.svg" alt="twitter icon" width="24"></a> |
| <a href="https://www.linkedin.com/company/apache-cassandra/" target="_blank" styles="margin-left: 20px;"><img src="../../assets/img/LI-In-Bug.png" alt="linked-in icon" width="24"></a> |
| <a href="https://www.youtube.com/c/PlanetCassandra" target="_blank" styles="margin-left: 20px;"><img src="../../assets/img/youtube-icon.png" alt="youtube icon" width="24"></a> |
| </div> |
| <div class="cf"> |
| <div class="logo left"><a href="/"><img src="../../assets/img/logo-white-r.png" alt="cassandra logo"></a></div> |
| <div class="mobile-nav-icon right"> |
| <img class="toggle-icon" src="../../assets/img/hamburger-nav.svg"> |
| </div> |
| <ul class="main-nav nav-links right flex flex-vert-center flex-space-between"> |
| <li> |
| <a class="nav-link hide-mobile">Get Started</a> |
| <ul class="sub-menu bg-white"> |
| <li class="pa-micro"> |
| <a href="/_/cassandra-basics.html"> |
| <div class="sub-nav-icon"> |
| <img src="../../assets/img/sub-menu-basics.png" alt="cassandra basics icon"> |
| </div> |
| <div class="sub-nav-text teal py-small"> |
| Cassandra Basics |
| </div> |
| </a> |
| </li> |
| <li class="pa-micro"> |
| <a href="/_/quickstart.html"> |
| <div class="sub-nav-icon"> |
| <img src="../../assets/img/sub-menu-rocket.png" alt="cassandra basics icon"> |
| </div> |
| <div class="sub-nav-text teal py-small"> |
| Quickstart |
| </div> |
| </a> |
| </li> |
| <li class="pa-micro"> |
| <a href="/_/ecosystem.html"> |
| <div class="sub-nav-icon"> |
| <img src="../../assets/img/sub-menu-ecosystem.png" alt="cassandra basics icon"> |
| </div> |
| <div class="sub-nav-text teal py-small"> |
| Ecosystem |
| </div> |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li><a class="nav-link" href="/doc/latest/">Documentation</a></li> |
| <li> |
| <a class="nav-link" href="/_/community.html">Community</a> |
| <ul class="sub-menu bg-white"> |
| <li class="pa-micro"> |
| <a href="/_/community.html#code-of-conduct"> |
| <div class="sub-nav-icon"> |
| <img src="../../assets/img/sub-menu-welcome.png" alt="welcome icon"> |
| </div> |
| <div class="sub-nav-text teal py-small"> |
| Welcome |
| </div> |
| </a> |
| </li> |
| <li class="pa-micro hide-mobile"> |
| <a href="/_/community.html#discussions"> |
| <div class="sub-nav-icon"> |
| <img src="../../assets/img/sub-menu-discussions.png" alt="discussions icon"> |
| </div> |
| <div class="sub-nav-text teal py-small"> |
| Discussions |
| </div> |
| </a> |
| </li> |
| <li class="pa-micro hide-mobile"> |
| <a href="/_/community.html#project-governance"> |
| <div class="sub-nav-icon"> |
| <img src="../../assets/img/sub-menu-governance.png" alt="Governance icon"> |
| </div> |
| <div class="sub-nav-text teal py-small"> |
| Governance |
| </div> |
| </a> |
| </li> |
| <li class="pa-micro hide-mobile"> |
| <a href="/_/community.html#how-to-contribute"> |
| <div class="sub-nav-icon"> |
| <img src="../../assets/img/sub-menu-contribute.png" alt="Contribute icon"> |
| </div> |
| <div class="sub-nav-text teal py-small"> |
| Contribute |
| </div> |
| </a> |
| </li> |
| <li class="pa-micro hide-mobile"> |
| <a href="/_/community.html#meet-the-community"> |
| <div class="sub-nav-icon"> |
| <img src="../../assets/img/sub-menu-community.png" alt="Meet the Community icon"> |
| </div> |
| <div class="sub-nav-text teal py-small"> |
| Meet the Community |
| </div> |
| </a> |
| </li> |
| <li class="pa-micro hide-mobile"> |
| <a href="/_/cassandra-catalyst-program.html"> |
| <div class="sub-nav-icon"> |
| <img src="../../assets/img/sub-menu-catalyst.png" alt="Catalyst icon"> |
| </div> |
| <div class="sub-nav-text teal py-small"> |
| Catalyst Program |
| </div> |
| </a> |
| </li> |
| <li class="pa-micro hide-mobile"> |
| <a href="/_/events.html"> |
| <div class="sub-nav-icon"> |
| <img src="../../assets/img/sub-menu-events.png" alt="Events icon"> |
| </div> |
| <div class="sub-nav-text teal py-small"> |
| Events |
| </div> |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li> |
| <a class="nav-link hide-mobile">Learn</a> |
| <ul class="sub-menu bg-white"> |
| <li class="pa-micro"> |
| <a href="/_/Apache-Cassandra-5.0-Moving-Toward-an-AI-Driven-Future.html"> |
| <div class="sub-nav-icon"> |
| <img src="../../assets/img/sub-menu-basics.png" alt="Basics icon"> |
| </div> |
| <div class="sub-nav-text teal py-small"> |
| Cassandra 5.0 |
| </div> |
| </a> |
| </li> |
| <li class="pa-micro"> |
| <a href="/_/case-studies.html"> |
| <div class="sub-nav-icon"> |
| <img src="../../assets/img/sub-menu-case-study.png" alt="Case Studies icon"> |
| </div> |
| <div class="sub-nav-text teal py-small"> |
| Case Studies |
| </div> |
| </a> |
| </li> |
| <li class="pa-micro"> |
| <a href="/_/resources.html"> |
| <div class="sub-nav-icon"> |
| <img src="../../assets/img/sub-menu-resources.png" alt="Resources icon"> |
| </div> |
| <div class="sub-nav-text teal py-small"> |
| Resources |
| </div> |
| </a> |
| </li> |
| <li class="pa-micro"> |
| <a href="/_/blog.html"> |
| <div class="sub-nav-icon"> |
| <img src="../../assets/img/sub-menu-blog.png" alt="Blog icon"> |
| </div> |
| <div class="sub-nav-text teal py-small"> |
| Blog |
| </div> |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li><a class="nav-link btn btn--filled" href="/_/download.html">Download Now</a></li> |
| </ul> |
| </div> |
| </div> |
| </header> |
| |
| <div class="hero hero--home grad"> |
| <div class="eye"></div> |
| <div id="home-content" class="text-center flex flex-center flex-column relative z2 ma-xlarge"> |
| <h1>Hardware-bound Zero Copy Streaming in Apache Cassandra 4.0</h1> |
| <h3>August 07, 2018 | The Apache Cassandra Community</h3> |
| </div> |
| </div> |
| <div id="blog-post" class="flex-center py-large arrow"> |
| <div class="blog-breadcrumb mb-medium"> |
| <div class="inner inner--narrow"> |
| <a href="/_/blog.html">« Back to the Apache Cassandra Blog</a> |
| </div> |
| </div> |
| <div class="post-content"> |
| <div class="inner inner--narrow"> |
| <div id="preamble"> |
| <div class="sectionbody"> |
| <div class="paragraph"> |
| <p>Streaming in Apache Cassandra powers host replacement, range movements, and cluster expansions. Streaming plays a crucial role in the cluster and as such its performance is key to not only the speed of the operations its used in but the cluster’s health generally. In Apache Cassandra 4.0, we have introduced an improved streaming implementation that reduces GC pressure and increases throughput several folds and are now limited, in some cases, only by the disk / network IO (See: <a href="https://issues.apache.org/jira/browse/CASSANDRA-14556" target="_blank" rel="noopener">CASSANDRA-14556</a>).</p> |
| </div> |
| <div class="paragraph"> |
| <p><span class="image right"><img src="../_images/blog/hardware-bound-1.png" alt="Fig 1. Cassandra Streaming" width="450" height="450"></span>To get an understanding of the impact of these changes, let’s first have a look at the current streaming code path. The diagram below illustrates the stream session setup when a node attempts to stream data from a peer. Let’s say, we have a 3 node cluster (Nodes A, B, C). Node C is being rebuilt and has to stream all data that it is responsible for from A & B. C setups a streaming session with each of it’s peers (See: <a href="https://issues.apache.org/jira/browse/CASSANDRA-4650" target="_blank" rel="noopener">CASSANDRA-4560</a> how Cassandra applies <a href="https://en.wikipedia.org/wiki/Ford%E2%80%93Fulkerson_algorithm" target="_blank" rel="noopener">Ford Fulkerson</a> to optimize streaming peers). It exchanges messages to request ranges and begins streaming data from the selected nodes.</p> |
| </div> |
| <div class="paragraph"> |
| <p>During the streaming phase, A collects all SSTables that have partitions in the requested ranges. It streams each SSTable by serializing individual partitions. Upon receiving the partition, node C reifies the data in memory and then writes it to disk. This is necessary to accurately transfer partitions from all possible SSTables for the requested ranges. This streaming path generates garbage and could be avoided in scenarios where all partitions within the SSTable need to be transmitted. This is common when you’re using LeveledCompactionStrategy or have enabled partitioning SSTables by token range (See: <a href="http://issues.apache.org/jira/browse/CASSANDRA-6696" target="_blank" rel="noopener">CASSANDRA-6696</a>), etc.</p> |
| </div> |
| <div class="paragraph"> |
| <p>To solve this problem <a href="http://issues.apache.org/jira/browse/CASSANDRA-14556" target="_blank" rel="noopener">CASSANDRA-14556</a> adds a Zero Copy streaming path. This significantly speeds up the transfer of SSTables and reduces garbage and unnecessary object creation. It modifies the streaming path to add additional information into the streaming header and uses ZeroCopy APIs to transfer bytes to and from the network and disk. So now, an SSTable may be transferred using this strategy when Cassandra detects that a complete SSTable needs to be transferred.</p> |
| </div> |
| </div> |
| </div> |
| <div class="sect1"> |
| <h2 id="how-do-i-use-this-feature"><a class="anchor" href="#how-do-i-use-this-feature"></a>How do I use this feature?</h2> |
| <div class="sectionbody"> |
| <div class="paragraph"> |
| <p>It just works. This feature is controlled using <code>stream_entire_sstables</code> in <code>cassandra.yaml</code> and is enabled by default. Even though this feature is enabled, it will respect the throttling limits as defined by <code>stream_throughput_outbound_megabits_per_sec</code>.</p> |
| </div> |
| </div> |
| </div> |
| <div class="sect1"> |
| <h2 id="impact"><a class="anchor" href="#impact"></a>Impact</h2> |
| <div class="sectionbody"> |
| <div class="paragraph"> |
| <p>Cassandra can stream SSTables only bounded by the hardware limitations (Network and Disk IO). With this optimization, we hope to make Cassandra more performant and reliable.</p> |
| </div> |
| <div class="paragraph"> |
| <p>Microbenchmarking this feature shows a marked improvement (higher is better). Block Stream Writers are the ZeroCopy writers and Partial Stream Writers are the existing writers.</p> |
| </div> |
| <table class="tableblock frame-all grid-all stretch"> |
| <colgroup> |
| <col style="width: 16.6666%;"> |
| <col style="width: 16.6666%;"> |
| <col style="width: 16.6666%;"> |
| <col style="width: 16.6666%;"> |
| <col style="width: 16.6666%;"> |
| <col style="width: 16.667%;"> |
| </colgroup> |
| <thead> |
| <tr> |
| <th class="tableblock halign-left valign-top">Benchmark</th> |
| <th class="tableblock halign-left valign-top">Mode</th> |
| <th class="tableblock halign-left valign-top">Cnt</th> |
| <th class="tableblock halign-left valign-top">Score</th> |
| <th class="tableblock halign-left valign-top">Error</th> |
| <th class="tableblock halign-left valign-top">Units</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">ZeroCopyStreamingBenchmark.blockStreamReader</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">thrpt</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">10</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">20.119</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">± 1.300</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">ops/s</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">ZeroCopyStreamingBenchmark.blockStreamWriter</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">thrpt</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">10</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">1339.672</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">± 352.242</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">ops/s</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">ZeroCopyStreamingBenchmark.partialStreamReader</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">thrpt</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">10</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">0.590</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">± 0.135</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">ops/s</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">ZeroCopyStreamingBenchmark.partialStreamWriter</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">thrpt</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">10</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">17.556</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">± 0.323</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">ops/s</p></td> |
| </tr> |
| </tbody> |
| </table> |
| </div> |
| </div> |
| <div class="sect1"> |
| <h2 id="conclusion"><a class="anchor" href="#conclusion"></a>Conclusion</h2> |
| <div class="sectionbody"> |
| <div class="paragraph"> |
| <p>If you’re a Cassandra user, we would love to hear back from you. Please send us feedback via user <a href="http://cassandra.apache.org/community/" target="_blank" rel="noopener">Mailing List</a>, <a href="https://issues.apache.org/jira/projects/CASSANDRA/summary" target="_blank" rel="noopener">Jira</a>, or <a href="http://cassandra.apache.org/community/" target="_blank" rel="noopener">IRC</a> (or any combination of the three).</p> |
| </div> |
| </div> |
| </div> |
| </div> |
| </div> |
| </div> |
| <footer class="grad grad--two flex-center pb-xlarge"> |
| <div class="inner text-center z2 relative"> |
| <h2 class="white py-small">Get started with Cassandra, fast.</h2> |
| <a id="footer-cta" href="/_/quickstart.html" class="btn btn--filled ma-medium">Quickstart Guide</a> |
| </div> |
| <div class="inner flex flex-distribute-items mt-xlarge z2 relative"> |
| <div class="col-2"> |
| <div id="footer-logo" class="logo logo--footer mb-medium"><img src="../../assets/img/logo-white-r.png" alt="Cassandra Logo"></div> |
| <p>Apache Cassandra<img src="../../assets/img/registered.svg" alt="®" style="width:18px;"> powers mission-critical deployments with improved performance and unparalleled levels of scale in the cloud.</p> |
| <div class="footer-social-icons"> |
| <a href="https://twitter.com/cassandra?lang=en" target="_blank"><img src="../../assets/img/twitter-icon-circle-white.svg" alt="twitter icon" width="24"></a> |
| <a href="https://www.linkedin.com/company/apache-cassandra/" target="_blank"><img src="../../assets/img/LI-In-Bug.png" alt="linked-in icon" width="24"></a> |
| <a href="https://www.youtube.com/c/PlanetCassandra" target="_blank"><img src="../../assets/img/youtube-icon.png" alt="youtube icon" width="24"></a> |
| </div> |
| </div> |
| <div class="col-2 flex flex-center"> |
| <ul class="columns-2"> |
| <li class="mb-small"><a href="/">Home</a></li> |
| <li class="mb-small"><a href="/_/cassandra-basics.html">Cassandra Basics</a></li> |
| <li class="mb-small"><a href="/_/quickstart.html">Quickstart</a></li> |
| <li class="mb-small"><a href="/_/ecosystem.html">Ecosystem</a></li> |
| <li class="mb-small"><a href="/doc/latest/">Documentation</a></li> |
| <li class="mb-small"><a href="/_/community.html">Community</a></li> |
| <li class="mb-small"><a href="/_/case-studies.html">Case Studies</a></li> |
| <li class="mb-small"><a href="/_/resources.html">Resources</a></li> |
| <li class="mb-small"><a href="/_/blog.html">Blog</a></li> |
| </ul> |
| </div> |
| </div> |
| </footer> |
| <div class="lower-footer bg-white pa-medium"> |
| <div class="flex flex-row flex-vert-center"> |
| <div class="pr-medium"><img src="../../assets/img//feather-small.png" alt="ASF" width="20"></div> |
| <div class="pr-medium"><a href="http://www.apache.org/" target="_blank">Foundation</a></div> |
| <div class="pr-medium"><a href="https://www.apache.org/events/current-event.html" target="_blank">Events</a></div> |
| <div class="pr-medium"><a href="https://www.apache.org/licenses/" target="_blank">License</a></div> |
| <div class="pr-medium"><a href="https://www.apache.org/foundation/thanks" target="_blank">Thanks</a></div> |
| <div class="pr-medium"><a href="https://www.apache.org/security" target="_blank">Security</a></div> |
| <div class="pr-medium"><a href="https://privacy.apache.org/policies/privacy-policy-public.html" target="_blank">Privacy</a></div> |
| <div class="pr-medium"><a href="https://www.apache.org/foundation/sponsorship" target="_blank">Sponsorship</a></div> |
| </div> |
| <p class="my-medium">© 2009-<script>document.write(new Date().getFullYear())</script> <a href="https://apache.org" target="_blank">The Apache Software Foundation</a> under the terms of the Apache License 2.0. Apache, the Apache feather logo, Apache Cassandra, Cassandra, and the Cassandra logo, are either registered trademarks or trademarks of The Apache Software Foundation.</p> |
| </div> |
| <div id="fade" class="hidden"></div> |
| <div id="modal" class="hidden"> |
| <div id="close-modal" class="cursor-pointer"><svg viewBox="0 0 24 24" width="24" height="24" stroke="currentColor" stroke-width="2" fill="none" stroke-linecap="round" stroke-linejoin="round" class="css-i6dzq1"><line x1="18" y1="6" x2="6" y2="18"></line><line x1="6" y1="6" x2="18" y2="18"></line></svg></div> |
| <div id="mod-content" class="vid-mod-content resp-container"></div> |
| </div> |
| <script> |
| jQuery(function(){ |
| var windowW = $(window).width(); |
| $(document) |
| .on('click','.mobile-nav-icon',function(){ |
| $('.main-nav').fadeIn(); |
| }) |
| .on('click','.main-nav',function(){ |
| if(windowW <= 1000){ |
| $(this).fadeOut(); |
| } |
| }) |
| .on('click','#version-toggle',function(){ |
| $(this).toggleClass('active'); |
| $(this).next().fadeToggle(); |
| }) |
| .on('click','#mobile-docs-nav-burger', function(){ |
| $(this).toggleClass('active'); |
| $('.docs-nav').toggleClass('active'); |
| }); |
| var url = window.location.pathname; |
| var isQuickstart = url.includes('quickstart.html'); |
| if(isQuickstart){ |
| var footerCTA = document.getElementById('footer-cta'); |
| footerCTA.innerHTML = 'Get latest updates'; |
| footerCTA.setAttribute('href', '/_/blog.html'); |
| } |
| }); |
| </script> |
| </div> |
| </body> |
| <script> |
| jQuery(function(){ |
| |
| }); |
| </script> |
| </html> |
