| <!DOCTYPE html> |
| <html lang="en"> |
| <head> |
| <meta charset="utf-8"> |
| <meta name="viewport" content="width=device-width,initial-scale=1.0"> |
| <title>Improved Internode Messaging | Apache Cassandra Documentation</title> |
| <link rel="stylesheet" href="../../../../assets/css/site.css"> |
| <link rel="schema.dcterms" href="https://purl.org/dc/terms/"> |
| <meta name="dcterms.subject" content="Cassandra"> |
| <meta name="dcterms.identifier" content="4.1"> |
| <meta name="generator" content="Antora 2.3.4"> |
| <link rel="icon" href="../../../../assets/img/favicon.ico" type="image/x-icon"> |
| <script> |
| const script = document.createElement("script"); |
| const domain = window.location.hostname; |
| script.type = "text/javascript"; |
| script.src = "https://plausible.cassandra.apache.org/js/plausible.js"; |
| script.setAttribute("data-domain",domain); |
| script.setAttribute("defer",'true'); |
| script.setAttribute("async",'true'); |
| document.getElementsByTagName("head")[0].appendChild(script); |
| </script> </head> |
| <body class="docs-wrapper article"> |
| <div class="container mx-auto relative"> |
| <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.6.0/jquery.min.js"></script> |
| <meta property="og:type" content="website" /> |
| <meta property="og:url" content="/" /> |
| <meta property="og:site_name" content="Apache Cassandra" /> |
| |
| <header id="top-nav"> |
| <div class="inner relative"> |
| <div class="header-social-icons text-right"> |
| <a href="https://twitter.com/cassandra?lang=en" target="_blank" styles="margin-left: 20px;"><img src="../../../../assets/img/twitter-icon-circle-white.svg" alt="twitter icon" width="24"></a> |
| <a href="https://www.linkedin.com/company/apache-cassandra/" target="_blank" styles="margin-left: 20px;"><img src="../../../../assets/img/LI-In-Bug.png" alt="linked-in icon" width="24"></a> |
| <a href="https://www.youtube.com/c/PlanetCassandra" target="_blank" styles="margin-left: 20px;"><img src="../../../../assets/img/youtube-icon.png" alt="youtube icon" width="24"></a> |
| </div> |
| <div class="cf"> |
| <div class="logo left"><a href="/"><img src="../../../../assets/img/logo-white-r.png" alt="Cassandra Logo"></a></div> |
| <div class="mobile-nav-icon right"> |
| <img class="toggle-icon" src="../../../../assets/img/hamburger-nav.svg"> |
| </div> |
| <ul class="main-nav nav-links right flex flex-vert-center flex-space-between"> |
| <li> |
| <a class="nav-link hide-mobile">Get Started</a> |
| <ul class="sub-menu bg-white"> |
| <li class="pa-micro"> |
| <a href="/_/cassandra-basics.html"> |
| <div class="sub-nav-icon"> |
| <img src="../../../../assets/img/sub-menu-basics.png" alt="cassandra basics icon"> |
| </div> |
| <div class="sub-nav-text teal py-small"> |
| Cassandra Basics |
| </div> |
| </a> |
| </li> |
| <li class="pa-micro"> |
| <a href="/_/quickstart.html"> |
| <div class="sub-nav-icon"> |
| <img src="../../../../assets/img/sub-menu-rocket.png" alt="cassandra basics icon"> |
| </div> |
| <div class="sub-nav-text teal py-small"> |
| Quickstart |
| </div> |
| </a> |
| </li> |
| <li class="pa-micro"> |
| <a href="/_/ecosystem.html"> |
| <div class="sub-nav-icon"> |
| <img src="../../../../assets/img/sub-menu-ecosystem.png" alt="cassandra basics icon"> |
| </div> |
| <div class="sub-nav-text teal py-small"> |
| Ecosystem |
| </div> |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li><a class="nav-link" href="/doc/latest/">Documentation</a></li> |
| <li> |
| <a class="nav-link" href="/_/community.html">Community</a> |
| <ul class="sub-menu bg-white"> |
| <li class="pa-micro"> |
| <a href="/_/community.html#code-of-conduct"> |
| <div class="sub-nav-icon"> |
| <img src="../../../../assets/img/sub-menu-welcome.png" alt="welcome icon"> |
| </div> |
| <div class="sub-nav-text teal py-small"> |
| Welcome |
| </div> |
| </a> |
| </li> |
| <li class="pa-micro hide-mobile"> |
| <a href="/_/community.html#discussions"> |
| <div class="sub-nav-icon"> |
| <img src="../../../../assets/img/sub-menu-discussions.png" alt="discussions icon"> |
| </div> |
| <div class="sub-nav-text teal py-small"> |
| Discussions |
| </div> |
| </a> |
| </li> |
| <li class="pa-micro hide-mobile"> |
| <a href="/_/community.html#project-governance"> |
| <div class="sub-nav-icon"> |
| <img src="../../../../assets/img/sub-menu-governance.png" alt="Governance icon"> |
| </div> |
| <div class="sub-nav-text teal py-small"> |
| Governance |
| </div> |
| </a> |
| </li> |
| <li class="pa-micro hide-mobile"> |
| <a href="/_/community.html#how-to-contribute"> |
| <div class="sub-nav-icon"> |
| <img src="../../../../assets/img/sub-menu-contribute.png" alt="Contribute icon"> |
| </div> |
| <div class="sub-nav-text teal py-small"> |
| Contribute |
| </div> |
| </a> |
| </li> |
| <li class="pa-micro hide-mobile"> |
| <a href="/_/community.html#meet-the-community"> |
| <div class="sub-nav-icon"> |
| <img src="../../../../assets/img/sub-menu-community.png" alt="Meet the Community icon"> |
| </div> |
| <div class="sub-nav-text teal py-small"> |
| Meet the Community |
| </div> |
| </a> |
| </li> |
| <li class="pa-micro hide-mobile"> |
| <a href="/_/cassandra-catalyst-program.html"> |
| <div class="sub-nav-icon"> |
| <img src="../../../../assets/img/sub-menu-catalyst.png" alt="Catalyst icon"> |
| </div> |
| <div class="sub-nav-text teal py-small"> |
| Catalyst Program |
| </div> |
| </a> |
| </li> |
| <li class="pa-micro hide-mobile"> |
| <a href="/_/events.html"> |
| <div class="sub-nav-icon"> |
| <img src="../../../../assets/img/sub-menu-events.png" alt="Events icon"> |
| </div> |
| <div class="sub-nav-text teal py-small"> |
| Events |
| </div> |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li> |
| <a class="nav-link hide-mobile">Learn</a> |
| <ul class="sub-menu bg-white"> |
| <li class="pa-micro"> |
| <a href="/_/Apache-Cassandra-5.0-Moving-Toward-an-AI-Driven-Future.html"> |
| <div class="sub-nav-icon"> |
| <img src="../../../../assets/img/sub-menu-basics.png" alt="Basics icon"> |
| </div> |
| <div class="sub-nav-text teal py-small"> |
| Cassandra 5.0 |
| </div> |
| </a> |
| </li> |
| <li class="pa-micro"> |
| <a href="/_/case-studies.html"> |
| <div class="sub-nav-icon"> |
| <img src="../../../../assets/img/sub-menu-case-study.png" alt="Case Studies icon"> |
| </div> |
| <div class="sub-nav-text teal py-small"> |
| Case Studies |
| </div> |
| </a> |
| </li> |
| <li class="pa-micro"> |
| <a href="/_/resources.html"> |
| <div class="sub-nav-icon"> |
| <img src="../../../../assets/img/sub-menu-resources.png" alt="Resources icon"> |
| </div> |
| <div class="sub-nav-text teal py-small"> |
| Resources |
| </div> |
| </a> |
| </li> |
| <li class="pa-micro"> |
| <a href="/_/blog.html"> |
| <div class="sub-nav-icon"> |
| <img src="../../../../assets/img/sub-menu-blog.png" alt="Blog icon"> |
| </div> |
| <div class="sub-nav-text teal py-small"> |
| Blog |
| </div> |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li><a class="nav-link btn btn--filled" href="/_/download.html">Download Now</a></li> |
| </ul> |
| </div> |
| </div> |
| </header> |
| |
| <div class="hero hero--home grad"> |
| <div class="eye"></div> |
| <div id="docs-content" class="text-center flex flex-center flex-column relative z2 ma-xlarge"> |
| <h2>Cassandra Documentation</h2> |
| </div> |
| </div> |
| <div class="body px-medium py-medium container"> |
| <div class="docs-nav-bar flex flex-space-between mb-medium"> |
| <div id="mobile-docs-nav-burger" class="hidden"> |
| <svg viewBox="0 0 24 24" width="36" height="36" stroke="#1c81a0" stroke-width="2.5" fill="none" stroke-linecap="round" stroke-linejoin="round" class="css-i6dzq1"><line x1="3" y1="12" x2="21" y2="12"></line><line x1="3" y1="6" x2="21" y2="6"></line><line x1="3" y1="18" x2="21" y2="18"></line></svg> |
| </div> |
| <div class="docs-nav-item relative"> |
| <input id="search-input" type="text" placeholder="Search docs"> |
| </div> |
| <div class="versions-wrapper"> |
| <h4>Version:</h4> |
| <div class="nav-panel-explore" data-panel="explore"> |
| |
| <div id="version-toggle" class="context"> |
| <span class="version">4.1</span> |
| </div> |
| <ul id="versions-list" class="components"> |
| <li class="component"> |
| <ul class="versions"> |
| <li class="version is-latest"> |
| <a href="../../../../_/index.html">master</a> |
| </li> |
| </ul> |
| </li> |
| <li class="component is-current"> |
| <ul class="versions"> |
| <li class="version"> |
| <a href="../../../trunk/index.html">trunk</a> |
| </li> |
| <li class="version"> |
| <a href="../../../5.0/index.html">5.0</a> |
| </li> |
| <li class="version is-current is-latest"> |
| <a href="../../index.html">4.1</a> |
| </li> |
| <li class="version"> |
| <a href="../../../4.0/index.html">4.0</a> |
| </li> |
| <li class="version"> |
| <a href="../../../3.11/index.html">3.11</a> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| </div> |
| </div> </div> |
| <div class="cf relative"> |
| <nav class="nav docs-nav full-800"> |
| <div class="nav-menu"> |
| <ul class="nav-list"> |
| <li class="nav-item is-active" data-depth="0"> |
| <ul class="nav-list"> |
| <li class="nav-item" data-depth="1"> |
| <span class="nav-line"> |
| <button class="nav-toggle"></button> |
| <a class="nav-link" href="../../index.html">Main</a> |
| </span> |
| <ul class="nav-list"> |
| <li class="nav-item" data-depth="2"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../../../../_/glossary.html">Glossary</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="2"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../../../../_/bugs.html">How to report bugs</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="2"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../../../../_/contactus.html">Contact us</a> |
| </span> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li class="nav-item is-active" data-depth="0"> |
| <ul class="nav-list"> |
| <li class="nav-item is-current-path is-active" data-depth="1"> |
| <span class="nav-line"> |
| <button class="nav-toggle"></button> |
| <span class="nav-text">Cassandra</span> |
| </span> |
| <ul class="nav-list"> |
| <li class="nav-item" data-depth="2"> |
| <span class="nav-line"> |
| <button class="nav-toggle"></button> |
| <a class="nav-link" href="../getting_started/index.html">Getting Started</a> |
| </span> |
| <ul class="nav-list"> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../getting_started/installing.html">Installing Cassandra</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../getting_started/configuring.html">Configuring Cassandra</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../getting_started/querying.html">Inserting and querying</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../getting_started/drivers.html">Client drivers</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../getting_started/java11.html">Support for Java 11</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../getting_started/production.html">Production recommendations</a> |
| </span> |
| </li> |
| </ul> |
| </li> |
| <li class="nav-item" data-depth="2"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../new/index.html">What’s new</a> |
| </span> |
| </li> |
| <li class="nav-item is-current-path is-active" data-depth="2"> |
| <span class="nav-line"> |
| <button class="nav-toggle"></button> |
| <a class="nav-link" href="index.html">Architecture</a> |
| </span> |
| <ul class="nav-list"> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="overview.html">Overview</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="dynamo.html">Dynamo</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="storage_engine.html">Storage engine</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="guarantees.html">Guarantees</a> |
| </span> |
| </li> |
| <li class="nav-item is-current-page is-active" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="messaging.html">Improved internode messaging</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="streaming.html">Improved streaming</a> |
| </span> |
| </li> |
| </ul> |
| </li> |
| <li class="nav-item" data-depth="2"> |
| <span class="nav-line"> |
| <button class="nav-toggle"></button> |
| <a class="nav-link" href="../data_modeling/index.html">Data modeling</a> |
| </span> |
| <ul class="nav-list"> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../data_modeling/intro.html">Introduction</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../data_modeling/data_modeling_conceptual.html">Conceptual data modeling</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../data_modeling/data_modeling_rdbms.html">RDBMS design</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../data_modeling/data_modeling_queries.html">Defining application queries</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../data_modeling/data_modeling_logical.html">Logical data modeling</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../data_modeling/data_modeling_physical.html">Physical data modeling</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../data_modeling/data_modeling_refining.html">Evaluating and refining data models</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../data_modeling/data_modeling_schema.html">Defining database schema</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../data_modeling/data_modeling_tools.html">Cassandra data modeling tools</a> |
| </span> |
| </li> |
| </ul> |
| </li> |
| <li class="nav-item" data-depth="2"> |
| <span class="nav-line"> |
| <button class="nav-toggle"></button> |
| <a class="nav-link" href="../cql/index.html">Cassandra Query Language (CQL)</a> |
| </span> |
| <ul class="nav-list"> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../cql/definitions.html">Definitions</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../cql/types.html">Data types</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../cql/ddl.html">Data definition (DDL)</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../cql/dml.html">Data manipulation (DML)</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../cql/operators.html">Operators</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../cql/indexes.html">Secondary indexes</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../cql/mvs.html">Materialized views</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../cql/functions.html">Functions</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../cql/json.html">JSON</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../cql/security.html">Security</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../cql/triggers.html">Triggers</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../cql/appendices.html">Appendices</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../cql/changes.html">Changes</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../cql/SASI.html">SASI</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../cql/cql_singlefile.html">Single file of CQL information</a> |
| </span> |
| </li> |
| </ul> |
| </li> |
| <li class="nav-item" data-depth="2"> |
| <span class="nav-line"> |
| <button class="nav-toggle"></button> |
| <a class="nav-link" href="../configuration/index.html">Configuration</a> |
| </span> |
| <ul class="nav-list"> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../configuration/cass_yaml_file.html">cassandra.yaml</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../configuration/cass_rackdc_file.html">cassandra-rackdc.properties</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../configuration/cass_env_sh_file.html">cassandra-env.sh</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../configuration/cass_topo_file.html">cassandra-topologies.properties</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../configuration/cass_cl_archive_file.html">commitlog-archiving.properties</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../configuration/cass_logback_xml_file.html">logback.xml</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../configuration/cass_jvm_options_file.html">jvm-* files</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../configuration/configuration.html">Liberating cassandra.yaml Parameters' Names from Their Units</a> |
| </span> |
| </li> |
| </ul> |
| </li> |
| <li class="nav-item" data-depth="2"> |
| <span class="nav-line"> |
| <button class="nav-toggle"></button> |
| <a class="nav-link" href="../operating/index.html">Operating</a> |
| </span> |
| <ul class="nav-list"> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="#operating/snitch.adoc">Snitches</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../operating/topo_changes.html">Topology changes</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../operating/repair.html">Repair</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../operating/read_repair.html">Read repair</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../operating/hints.html">Hints</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../operating/bloom_filters.html">Bloom filters</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../operating/compression.html">Compression</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../operating/cdc.html">Change Data Capture (CDC)</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../operating/backups.html">Backups</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../operating/bulk_loading.html">Bulk loading</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../operating/metrics.html">Metrics</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../operating/security.html">Security</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../operating/hardware.html">Hardware</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../operating/compaction/index.html">Compaction</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../operating/virtualtables.html">Virtual tables</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../operating/auditlogging.html">Audit logging</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../operating/audit_logging.html">Audit logging 2</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../operating/fqllogging.html">Full query logging</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../operating/transientreplication.html">Transient replication</a> |
| </span> |
| </li> |
| </ul> |
| </li> |
| <li class="nav-item" data-depth="2"> |
| <span class="nav-line"> |
| <button class="nav-toggle"></button> |
| <a class="nav-link" href="../tools/index.html">Tools</a> |
| </span> |
| <ul class="nav-list"> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../tools/cqlsh.html">cqlsh: the CQL shell</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../tools/nodetool/nodetool.html">nodetool</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../tools/sstable/index.html">SSTable tools</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../tools/cassandra_stress.html">cassandra-stress</a> |
| </span> |
| </li> |
| </ul> |
| </li> |
| <li class="nav-item" data-depth="2"> |
| <span class="nav-line"> |
| <button class="nav-toggle"></button> |
| <a class="nav-link" href="../troubleshooting/index.html">Troubleshooting</a> |
| </span> |
| <ul class="nav-list"> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../troubleshooting/finding_nodes.html">Finding misbehaving nodes</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../troubleshooting/reading_logs.html">Reading Cassandra logs</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../troubleshooting/use_nodetool.html">Using nodetool</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../troubleshooting/use_tools.html">Using external tools to deep-dive</a> |
| </span> |
| </li> |
| </ul> |
| </li> |
| <li class="nav-item" data-depth="2"> |
| <span class="nav-line"> |
| <button class="nav-toggle"></button> |
| <a class="nav-link" href="../../../../_/development/index.html">Development</a> |
| </span> |
| <ul class="nav-list"> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../../../../_/development/gettingstarted.html">Getting started</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../../../../_/development/ide.html">Building and IDE integration</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../../../../_/development/testing.html">Testing</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../../../../_/development/patches.html">Contributing code changes</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../../../../_/development/code_style.html">Code style</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../../../../_/development/how_to_review.html">Review checklist</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../../../../_/development/how_to_commit.html">How to commit</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../../../../_/development/documentation.html">Working on documentation</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../../../../_/development/ci.html">Jenkins CI environment</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../../../../_/development/dependencies.html">Dependency management</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="3"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../../../../_/development/release_process.html">Release process</a> |
| </span> |
| </li> |
| </ul> |
| </li> |
| <li class="nav-item" data-depth="2"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../faq/index.html">FAQ</a> |
| </span> |
| </li> |
| <li class="nav-item" data-depth="2"> |
| <span class="nav-line"> |
| <a class="nav-link" href="../plugins/index.html">Plug-ins</a> |
| </span> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| </div> |
| </nav> |
| <aside class="toc sidebar"> |
| <div class="toc-menu"></div> |
| </aside> |
| <main class="article default-main full-800" data-ceiling="topbar"> |
| <div class="article-header"> |
| <nav class="crumbs" aria-label="breadcrumbs"> |
| <ul> |
| <li class="crumb">Cassandra</li> |
| <li class="crumb"><a href="index.html">Architecture</a></li> |
| <li class="crumb"><a href="messaging.html">Improved internode messaging</a></li> |
| </ul> |
| </nav> |
| <div class="tools" role="navigation"> |
| <ul> |
| <li class="tool edit"><a href="https://github.com/apache/cassandra/edit/cassandra-4.1/doc/modules/cassandra/pages/architecture/messaging.adoc" title="Edit Page" target="_blank" rel="noopener">Edit</a></li> |
| </ul> |
| </div> |
| </div> |
| <article class="doc"> |
| <h1 class="page">Improved Internode Messaging</h1> |
| <div id="preamble"> |
| <div class="sectionbody"> |
| <div class="paragraph"> |
| <p>Apache Cassandra 4.0 has added several new improvements to internode |
| messaging.</p> |
| </div> |
| </div> |
| </div> |
| <div class="sect1"> |
| <h2 id="optimized-internode-messaging-protocol"><a class="anchor" href="#optimized-internode-messaging-protocol"></a>Optimized Internode Messaging Protocol</h2> |
| <div class="sectionbody"> |
| <div class="paragraph"> |
| <p>The internode messaging protocol has been optimized |
| (<a href="https://issues.apache.org/jira/browse/CASSANDRA-14485">CASSANDRA-14485</a>). |
| Previously the <code>IPAddressAndPort</code> of the sender was included with each |
| message that was sent even though the <code>IPAddressAndPort</code> had already |
| been sent once when the initial connection/session was established. In |
| Cassandra 4.0 <code>IPAddressAndPort</code> has been removed from every separate |
| message sent and only sent when connection/session is initiated.</p> |
| </div> |
| <div class="paragraph"> |
| <p>Another improvement is that at several instances (listed) a fixed 4-byte |
| integer value has been replaced with <code>vint</code> as a <code>vint</code> is almost always |
| less than 1 byte:</p> |
| </div> |
| <div class="ulist"> |
| <ul> |
| <li> |
| <p>The <code>paramSize</code> (the number of parameters in the header)</p> |
| </li> |
| <li> |
| <p>Each individual parameter value</p> |
| </li> |
| <li> |
| <p>The <code>payloadSize</code></p> |
| </li> |
| </ul> |
| </div> |
| </div> |
| </div> |
| <div class="sect1"> |
| <h2 id="nio-messaging"><a class="anchor" href="#nio-messaging"></a>NIO Messaging</h2> |
| <div class="sectionbody"> |
| <div class="paragraph"> |
| <p>In Cassandra 4.0 peer-to-peer (internode) messaging has been switched to |
| non-blocking I/O (NIO) via Netty |
| (<a href="https://issues.apache.org/jira/browse/CASSANDRA-8457">CASSANDRA-8457</a>).</p> |
| </div> |
| <div class="paragraph"> |
| <p>As serialization format, each message contains a header with several |
| fixed fields, an optional key-value parameters section, and then the |
| message payload itself. Note: the IP address in the header may be either |
| IPv4 (4 bytes) or IPv6 (16 bytes).</p> |
| </div> |
| <div class="quoteblock"> |
| <blockquote> |
| <div class="paragraph"> |
| <p>The diagram below shows the IPv4 address for brevity.</p> |
| </div> |
| </blockquote> |
| </div> |
| <div class="literalblock"> |
| <div class="content"> |
| <pre>1 1 1 1 1 2 2 2 2 2 3 3 3 3 3 4 4 4 4 4 5 5 5 5 5 6 6 |
| 0 2 4 6 8 0 2 4 6 8 0 2 4 6 8 0 2 4 6 8 0 2 4 6 8 0 2 4 6 8 0 2 |
| +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |
| | PROTOCOL MAGIC | |
| +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |
| | Message ID | |
| +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |
| | Timestamp | |
| +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |
| | Addr len | IP Address (IPv4) / |
| +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |
| / | Verb / |
| +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |
| / | Parameters size / |
| +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |
| / | Parameter data / |
| +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |
| / | |
| +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |
| | Payload size | |
| +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |
| | / |
| / Payload / |
| / | |
| +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+</pre> |
| </div> |
| </div> |
| <div class="paragraph"> |
| <p>An individual parameter has a String key and a byte array value. The key |
| is serialized with its length, encoded as two bytes, followed by the |
| UTF-8 byte encoding of the string. The body is serialized with its |
| length, encoded as four bytes, followed by the bytes of the value.</p> |
| </div> |
| </div> |
| </div> |
| <div class="sect1"> |
| <h2 id="resource-limits-on-queued-messages"><a class="anchor" href="#resource-limits-on-queued-messages"></a>Resource limits on Queued Messages</h2> |
| <div class="sectionbody"> |
| <div class="paragraph"> |
| <p>System stability is improved by enforcing strict resource limits |
| (<a href="https://issues.apache.org/jira/browse/CASSANDRA-15066">CASSANDRA-15066</a>) |
| on the number of outbound messages that are queued, measured by the |
| <code>serializedSize</code> of the message. There are three separate limits imposed |
| simultaneously to ensure that progress is always made without any |
| reasonable combination of failures impacting a node’s stability.</p> |
| </div> |
| <div class="olist arabic"> |
| <ol class="arabic"> |
| <li> |
| <p>Global, per-endpoint and per-connection limits are imposed on messages |
| queued for delivery to other nodes and waiting to be processed on |
| arrival from other nodes in the cluster. These limits are applied to the |
| on-wire size of the message being sent or received.</p> |
| </li> |
| <li> |
| <p>The basic per-link limit is consumed in isolation before any endpoint |
| or global limit is imposed. Each node-pair has three links: urgent, |
| small and large. Any given node may have a maximum of |
| <code>N*3 * (internode_application_send_queue_capacity in bytes + internode_application_receive_queue_capacity in bytes)</code> |
| of messages data queued without any coordination between them although in |
| practice, with token-aware routing, only RF*tokens nodes should need to |
| communicate with significant bandwidth.</p> |
| </li> |
| <li> |
| <p>The per-endpoint limit is imposed on all messages exceeding the |
| per-link limit, simultaneously with the global limit, on all links to or |
| from a single node in the cluster. The global limit is imposed on all |
| messages exceeding the per-link limit, simultaneously with the |
| per-endpoint limit, on all links to or from any node in the cluster. The |
| following configuration settings have been added to <code>cassandra.yaml</code> for |
| resource limits on queued messages.</p> |
| </li> |
| </ol> |
| </div> |
| <div class="literalblock"> |
| <div class="content"> |
| <pre>internode_application_send_queue_capacity: 4MiB |
| internode_application_send_queue_reserve_endpoint_capacity: 128MiB |
| internode_application_send_queue_reserve_global_capacity: 512MiB |
| internode_application_receive_queue_capacity: 4MiB |
| internode_application_receive_queue_reserve_endpoint_capacity: 128MiB |
| internode_application_receive_queue_reserve_global_capacity: 512MiB</pre> |
| </div> |
| </div> |
| </div> |
| </div> |
| <div class="sect1"> |
| <h2 id="virtual-tables-for-messaging-metrics"><a class="anchor" href="#virtual-tables-for-messaging-metrics"></a>Virtual Tables for Messaging Metrics</h2> |
| <div class="sectionbody"> |
| <div class="paragraph"> |
| <p>Metrics is improved by keeping metrics using virtual tables for |
| inter-node inbound and outbound messaging |
| (<a href="https://issues.apache.org/jira/browse/CASSANDRA-15066">CASSANDRA-15066</a>). |
| For inbound messaging a virtual table (<code>internode_inbound</code>) has been |
| added to keep metrics for:</p> |
| </div> |
| <div class="ulist"> |
| <ul> |
| <li> |
| <p>Bytes and count of messages that could not be serialized or flushed |
| due to an error</p> |
| </li> |
| <li> |
| <p>Bytes and count of messages scheduled</p> |
| </li> |
| <li> |
| <p>Bytes and count of messages successfully processed</p> |
| </li> |
| <li> |
| <p>Bytes and count of messages successfully received</p> |
| </li> |
| <li> |
| <p>Nanos and count of messages throttled</p> |
| </li> |
| <li> |
| <p>Bytes and count of messages expired</p> |
| </li> |
| <li> |
| <p>Corrupt frames recovered and unrecovered</p> |
| </li> |
| </ul> |
| </div> |
| <div class="paragraph"> |
| <p>A separate virtual table (<code>internode_outbound</code>) has been added for |
| outbound inter-node messaging. The outbound virtual table keeps metrics |
| for:</p> |
| </div> |
| <div class="ulist"> |
| <ul> |
| <li> |
| <p>Bytes and count of messages pending</p> |
| </li> |
| <li> |
| <p>Bytes and count of messages sent</p> |
| </li> |
| <li> |
| <p>Bytes and count of messages expired</p> |
| </li> |
| <li> |
| <p>Bytes and count of messages that could not be sent due to an error</p> |
| </li> |
| <li> |
| <p>Bytes and count of messages overloaded</p> |
| </li> |
| <li> |
| <p>Active Connection Count</p> |
| </li> |
| <li> |
| <p>Connection Attempts</p> |
| </li> |
| <li> |
| <p>Successful Connection Attempts</p> |
| </li> |
| </ul> |
| </div> |
| </div> |
| </div> |
| <div class="sect1"> |
| <h2 id="hint-messaging"><a class="anchor" href="#hint-messaging"></a>Hint Messaging</h2> |
| <div class="sectionbody"> |
| <div class="paragraph"> |
| <p>A specialized version of hint message that takes an already encoded in a |
| <code>ByteBuffer</code> hint and sends it verbatim has been added. It is an |
| optimization for when dispatching a hint file of the current messaging |
| version to a node of the same messaging version, which is the most |
| common case. It saves on extra <code>ByteBuffer</code> allocations one redundant |
| hint deserialization-serialization cycle.</p> |
| </div> |
| </div> |
| </div> |
| <div class="sect1"> |
| <h2 id="internode-application-timeout"><a class="anchor" href="#internode-application-timeout"></a>Internode Application Timeout</h2> |
| <div class="sectionbody"> |
| <div class="paragraph"> |
| <p>A configuration setting has been added to <code>cassandra.yaml</code> for the |
| maximum continuous period a connection may be unwritable in application |
| space.</p> |
| </div> |
| <div class="literalblock"> |
| <div class="content"> |
| <pre># internode_application_timeout_in_ms = 30000</pre> |
| </div> |
| </div> |
| <div class="paragraph"> |
| <p>Some other new features include logging of message size to trace message |
| for tracing a query.</p> |
| </div> |
| </div> |
| </div> |
| <div class="sect1"> |
| <h2 id="paxos-prepare-and-propose-stage-for-local-requests-optimized"><a class="anchor" href="#paxos-prepare-and-propose-stage-for-local-requests-optimized"></a>Paxos prepare and propose stage for local requests optimized</h2> |
| <div class="sectionbody"> |
| <div class="paragraph"> |
| <p>In pre-4.0 Paxos prepare and propose messages always go through entire |
| <code>MessagingService</code> stack in Cassandra even if request is to be served |
| locally, we can enhance and make local requests severed w/o involving |
| <code>MessagingService</code>. Similar things are done elsewhere in Cassandra which |
| skips <code>MessagingService</code> stage for local requests.</p> |
| </div> |
| <div class="paragraph"> |
| <p>This is what it looks like in pre 4.0 if we have tracing on and run a |
| light-weight transaction:</p> |
| </div> |
| <div class="literalblock"> |
| <div class="content"> |
| <pre>Sending PAXOS_PREPARE message to /A.B.C.D [MessagingService-Outgoing-/A.B.C.D] | 2017-09-11 |
| 21:55:18.971000 | A.B.C.D | 15045 |
| … REQUEST_RESPONSE message received from /A.B.C.D [MessagingService-Incoming-/A.B.C.D] | |
| 2017-09-11 21:55:18.976000 | A.B.C.D | 20270 |
| … Processing response from /A.B.C.D [SharedPool-Worker-4] | 2017-09-11 21:55:18.976000 | |
| A.B.C.D | 20372</pre> |
| </div> |
| </div> |
| <div class="paragraph"> |
| <p>Same thing applies for Propose stage as well.</p> |
| </div> |
| <div class="paragraph"> |
| <p>In version 4.0 Paxos prepare and propose stage for local requests are |
| optimized |
| (<a href="https://issues.apache.org/jira/browse/CASSANDRA-13862">CASSANDRA-13862</a>).</p> |
| </div> |
| </div> |
| </div> |
| <div class="sect1"> |
| <h2 id="quality-assurance"><a class="anchor" href="#quality-assurance"></a>Quality Assurance</h2> |
| <div class="sectionbody"> |
| <div class="paragraph"> |
| <p>Several other quality assurance improvements have been made in version |
| 4.0 |
| (<a href="https://issues.apache.org/jira/browse/CASSANDRA-15066">CASSANDRA-15066</a>).</p> |
| </div> |
| <div class="sect2"> |
| <h3 id="framing"><a class="anchor" href="#framing"></a>Framing</h3> |
| <div class="paragraph"> |
| <p>Version 4.0 introduces framing to all internode messages, i.e. the |
| grouping of messages into a single logical payload with headers and |
| trailers; these frames are guaranteed to either contain at most one |
| message, that is split into its own unique sequence of frames (for large |
| messages), or that a frame contains only complete messages.</p> |
| </div> |
| </div> |
| <div class="sect2"> |
| <h3 id="corruption-prevention"><a class="anchor" href="#corruption-prevention"></a>Corruption prevention</h3> |
| <div class="paragraph"> |
| <p>Previously, intra-datacenter internode messages would be unprotected |
| from corruption by default, as only LZ4 provided any integrity checks. |
| All messages to post 4.0 nodes are written to explicit frames, which may |
| be:</p> |
| </div> |
| <div class="ulist"> |
| <ul> |
| <li> |
| <p>LZ4 encoded</p> |
| </li> |
| <li> |
| <p>CRC protected</p> |
| </li> |
| </ul> |
| </div> |
| <div class="paragraph"> |
| <p>The Unprotected option is still available.</p> |
| </div> |
| </div> |
| <div class="sect2"> |
| <h3 id="resilience"><a class="anchor" href="#resilience"></a>Resilience</h3> |
| <div class="paragraph"> |
| <p>For resilience, all frames are written with a separate CRC protected |
| header, of 8 and 6 bytes respectively. If corruption occurs in this |
| header, the connection must be reset, as before. If corruption occurs |
| anywhere outside of the header, the corrupt frame will be skipped, |
| leaving the connection intact and avoiding the loss of any messages |
| unnecessarily.</p> |
| </div> |
| <div class="paragraph"> |
| <p>Previously, any issue at any point in the stream would result in the |
| connection being reset, with the loss of any in-flight messages.</p> |
| </div> |
| </div> |
| <div class="sect2"> |
| <h3 id="efficiency"><a class="anchor" href="#efficiency"></a>Efficiency</h3> |
| <div class="paragraph"> |
| <p>The overall memory usage, and number of byte shuffles, on both inbound |
| and outbound messages is reduced.</p> |
| </div> |
| <div class="paragraph"> |
| <p>Outbound the Netty LZ4 encoder maintains a chunk size buffer (64KiB), |
| that is filled before any compressed frame can be produced. Our frame |
| encoders avoid this redundant copy, as well as freeing 192KiB per |
| endpoint.</p> |
| </div> |
| <div class="paragraph"> |
| <p>Inbound, frame decoders guarantee only to copy the number of bytes |
| necessary to parse a frame, and to never store more bytes than |
| necessary. This improvement applies twice to LZ4 connections, improving |
| both the message decode and the LZ4 frame decode.</p> |
| </div> |
| </div> |
| <div class="sect2"> |
| <h3 id="inbound-path"><a class="anchor" href="#inbound-path"></a>Inbound Path</h3> |
| <div class="paragraph"> |
| <p>Version 4.0 introduces several improvements to the inbound path.</p> |
| </div> |
| <div class="paragraph"> |
| <p>An appropriate message handler is used based on whether large or small |
| messages are expected on a particular connection as set in a flag. |
| <code>NonblockingBufferHandler</code>, running on event loop, is used for small |
| messages, and <code>BlockingBufferHandler</code>, running off event loop, for large |
| messages. The single implementation of <code>InboundMessageHandler</code> handles |
| messages of any size effectively by deriving size of the incoming |
| message from the byte stream. In addition to deriving size of the |
| message from the stream, incoming message expiration time is proactively |
| read, before attempting to deserialize the entire message. If it’s |
| expired at the time when a message is encountered the message is just |
| skipped in the byte stream altogether. And if a message fails to be |
| deserialized while still on the receiving side - say, because of table |
| id or column being unknown - bytes are skipped, without dropping the |
| entire connection and losing all the buffered messages. An immediately |
| reply back is sent to the coordinator node with the failure reason, |
| rather than waiting for the coordinator callback to expire. This logic |
| is extended to a corrupted frame; a corrupted frame is safely skipped |
| over without dropping the connection.</p> |
| </div> |
| <div class="paragraph"> |
| <p>Inbound path imposes strict limits on memory utilization. Specifically, |
| the memory occupied by all parsed, but unprocessed messages is bound - |
| on per-connection, per-endpoint, and global basis. Once a connection |
| exceeds its local unprocessed capacity and cannot borrow any permits |
| from per-endpoint and global reserve, it simply stops processing further |
| messages, providing natural backpressure - until sufficient capacity is |
| regained.</p> |
| </div> |
| </div> |
| <div class="sect2"> |
| <h3 id="outbound-connections"><a class="anchor" href="#outbound-connections"></a>Outbound Connections</h3> |
| <div class="sect3"> |
| <h4 id="opening-a-connection"><a class="anchor" href="#opening-a-connection"></a>Opening a connection</h4> |
| <div class="paragraph"> |
| <p>A consistent approach is adopted for all kinds of failure to connect, |
| including: refused by endpoint, incompatible versions, or unexpected |
| exceptions;</p> |
| </div> |
| <div class="ulist"> |
| <ul> |
| <li> |
| <p>Retry forever, until either success or no messages waiting to deliver.</p> |
| </li> |
| <li> |
| <p>Wait incrementally longer periods before reconnecting, up to a maximum |
| of 1s.</p> |
| </li> |
| <li> |
| <p>While failing to connect, no reserve queue limits are acquired.</p> |
| </li> |
| </ul> |
| </div> |
| </div> |
| <div class="sect3"> |
| <h4 id="closing-a-connection"><a class="anchor" href="#closing-a-connection"></a>Closing a connection</h4> |
| <div class="ulist"> |
| <ul> |
| <li> |
| <p>Correctly drains outbound messages that are waiting to be delivered |
| (unless disconnected and fail to reconnect).</p> |
| </li> |
| <li> |
| <p>Messages written to a closing connection are either delivered or |
| rejected, with a new connection being opened if the old is irrevocably |
| closed.</p> |
| </li> |
| <li> |
| <p>Unused connections are pruned eventually.</p> |
| </li> |
| </ul> |
| </div> |
| </div> |
| <div class="sect3"> |
| <h4 id="reconnecting"><a class="anchor" href="#reconnecting"></a>Reconnecting</h4> |
| <div class="paragraph"> |
| <p>We sometimes need to reconnect a perfectly valid connection, e.g. if the |
| preferred IP address changes. We ensure that the underlying connection |
| has no in-progress operations before closing it and reconnecting.</p> |
| </div> |
| </div> |
| <div class="sect3"> |
| <h4 id="message-failure"><a class="anchor" href="#message-failure"></a>Message Failure</h4> |
| <div class="paragraph"> |
| <p>Propagates to callbacks instantly, better preventing overload by |
| reclaiming committed memory.</p> |
| </div> |
| <div class="sect4"> |
| <h5 id="expiry"><a class="anchor" href="#expiry"></a>Expiry</h5> |
| <div class="ulist"> |
| <ul> |
| <li> |
| <p>No longer experiences head-of-line blocking (e.g. undroppable message |
| preventing all droppable messages from being expired).</p> |
| </li> |
| <li> |
| <p>While overloaded, expiry is attempted eagerly on enqueuing threads.</p> |
| </li> |
| <li> |
| <p>While disconnected we schedule regular pruning, to handle the case |
| where messages are no longer being sent, but we have a large backlog to |
| expire.</p> |
| </li> |
| </ul> |
| </div> |
| </div> |
| <div class="sect4"> |
| <h5 id="overload"><a class="anchor" href="#overload"></a>Overload</h5> |
| <div class="ulist"> |
| <ul> |
| <li> |
| <p>Tracked by bytes queued, as opposed to number of messages.</p> |
| </li> |
| </ul> |
| </div> |
| </div> |
| <div class="sect4"> |
| <h5 id="serialization-errors"><a class="anchor" href="#serialization-errors"></a>Serialization Errors</h5> |
| <div class="ulist"> |
| <ul> |
| <li> |
| <p>Do not result in the connection being invalidated; the message is |
| simply completed with failure, and then erased from the frame.</p> |
| </li> |
| <li> |
| <p>Includes detected mismatch between calculated serialization size to |
| actual.</p> |
| </li> |
| </ul> |
| </div> |
| <div class="paragraph"> |
| <p>Failures to flush to network, perhaps because the connection has been |
| reset are not currently notified to callback handlers, as the necessary |
| information has been discarded, though it would be possible to do so in |
| future if we decide it is worth our while.</p> |
| </div> |
| </div> |
| </div> |
| <div class="sect3"> |
| <h4 id="qos"><a class="anchor" href="#qos"></a>QoS</h4> |
| <div class="paragraph"> |
| <p>"Gossip" connection has been replaced with a general purpose "Urgent" |
| connection, for any small messages impacting system stability.</p> |
| </div> |
| </div> |
| <div class="sect3"> |
| <h4 id="metrics"><a class="anchor" href="#metrics"></a>Metrics</h4> |
| <div class="paragraph"> |
| <p>We track, and expose via Virtual Table and JMX, the number of messages |
| and bytes that: we could not serialize or flush due to an error, we |
| dropped due to overload or timeout, are pending, and have successfully |
| sent.</p> |
| </div> |
| </div> |
| </div> |
| </div> |
| </div> |
| <div class="sect1"> |
| <h2 id="added-a-message-size-limit"><a class="anchor" href="#added-a-message-size-limit"></a>Added a Message size limit</h2> |
| <div class="sectionbody"> |
| <div class="paragraph"> |
| <p>Cassandra pre-4.0 doesn’t protect the server from allocating huge |
| buffers for the inter-node Message objects. Adding a message size limit |
| would be good to deal with issues such as a malfunctioning cluster |
| participant. Version 4.0 introduced max message size config param, akin |
| to max mutation size - set to endpoint reserve capacity by default.</p> |
| </div> |
| </div> |
| </div> |
| <div class="sect1"> |
| <h2 id="recover-from-unknown-table-when-deserializing-internode-messages"><a class="anchor" href="#recover-from-unknown-table-when-deserializing-internode-messages"></a>Recover from unknown table when deserializing internode messages</h2> |
| <div class="sectionbody"> |
| <div class="paragraph"> |
| <p>As discussed in |
| (<a href="https://issues.apache.org/jira/browse/CASSANDRA-9289">CASSANDRA-9289</a>) |
| it would be nice to gracefully recover from seeing an unknown table in a |
| message from another node. Pre-4.0, we close the connection and |
| reconnect, which can cause other concurrent queries to fail. Version 4.0 |
| fixes the issue by wrapping message in-stream with |
| <code>TrackedDataInputPlus</code>, catching <code>UnknownCFException</code>, and skipping the |
| remaining bytes in this message. TCP won’t be closed and it will remain |
| connected for other messages.</p> |
| </div> |
| </div> |
| </div> |
| </article> |
| </main> |
| </div> |
| </div> |
| <footer class="grad grad--two flex-center pb-xlarge"> |
| <div class="inner text-center z2 relative"> |
| <h2 class="white py-small">Get started with Cassandra, fast.</h2> |
| <a id="footer-cta" href="/_/quickstart.html" class="btn btn--filled ma-medium">Quickstart Guide</a> |
| </div> |
| <div class="inner flex flex-distribute-items mt-xlarge z2 relative"> |
| <div class="col-2"> |
| <div id="footer-logo" class="logo logo--footer mb-medium"><img src="../../../../assets/img/logo-white-r.png" alt="Cassandra Logo"></div> |
| <p>Apache Cassandra<img src="../../../../assets/img/registered.svg" alt="®" style="width:18px;"> powers mission-critical deployments with improved performance and unparalleled levels of scale in the cloud.</p> |
| <div class="footer-social-icons"> |
| <a href="https://twitter.com/cassandra?lang=en" target="_blank"><img src="../../../../assets/img/twitter-icon-circle-white.svg" alt="twitter icon" width="24"></a> |
| <a href="https://www.linkedin.com/company/apache-cassandra/" target="_blank"><img src="../../../../assets/img/LI-In-Bug.png" alt="linked-in icon" width="24"></a> |
| <a href="https://www.youtube.com/c/PlanetCassandra" target="_blank"><img src="../../../../assets/img/youtube-icon.png" alt="youtube icon" width="24"></a> |
| </div> |
| </div> |
| <div class="col-2 flex flex-center"> |
| <ul class="columns-2"> |
| <li class="mb-small"><a href="/">Home</a></li> |
| <li class="mb-small"><a href="/_/cassandra-basics.html">Cassandra Basics</a></li> |
| <li class="mb-small"><a href="/_/quickstart.html">Quickstart</a></li> |
| <li class="mb-small"><a href="/_/ecosystem.html">Ecosystem</a></li> |
| <li class="mb-small"><a href="/doc/latest/">Documentation</a></li> |
| <li class="mb-small"><a href="/_/community.html">Community</a></li> |
| <li class="mb-small"><a href="/_/case-studies.html">Case Studies</a></li> |
| <li class="mb-small"><a href="/_/resources.html">Resources</a></li> |
| <li class="mb-small"><a href="/_/blog.html">Blog</a></li> |
| </ul> |
| </div> |
| </div> |
| </footer> |
| <div class="lower-footer bg-white pa-medium"> |
| <div class="flex flex-row flex-vert-center"> |
| <div class="pr-medium"><img src="../../../../assets/img//feather-small.png" alt="ASF" width="20"></div> |
| <div class="pr-medium"><a href="http://www.apache.org/" target="_blank">Foundation</a></div> |
| <div class="pr-medium"><a href="https://www.apache.org/events/current-event.html" target="_blank">Events</a></div> |
| <div class="pr-medium"><a href="https://www.apache.org/licenses/" target="_blank">License</a></div> |
| <div class="pr-medium"><a href="https://www.apache.org/foundation/thanks" target="_blank">Thanks</a></div> |
| <div class="pr-medium"><a href="https://www.apache.org/security" target="_blank">Security</a></div> |
| <div class="pr-medium"><a href="https://privacy.apache.org/policies/privacy-policy-public.html" target="_blank">Privacy</a></div> |
| <div class="pr-medium"><a href="https://www.apache.org/foundation/sponsorship" target="_blank">Sponsorship</a></div> |
| </div> |
| <p class="my-medium">© 2009-<script>document.write(new Date().getFullYear())</script> <a href="https://apache.org" target="_blank">The Apache Software Foundation</a> under the terms of the Apache License 2.0. Apache, the Apache feather logo, Apache Cassandra, Cassandra, and the Cassandra logo, are either registered trademarks or trademarks of The Apache Software Foundation.</p> |
| </div> |
| <div id="fade" class="hidden"></div> |
| <div id="modal" class="hidden"> |
| <div id="close-modal" class="cursor-pointer"><svg viewBox="0 0 24 24" width="24" height="24" stroke="currentColor" stroke-width="2" fill="none" stroke-linecap="round" stroke-linejoin="round" class="css-i6dzq1"><line x1="18" y1="6" x2="6" y2="18"></line><line x1="6" y1="6" x2="18" y2="18"></line></svg></div> |
| <div id="mod-content" class="vid-mod-content resp-container"></div> |
| </div> |
| <script src="../../../../assets/js/site.js"></script> |
| <script async src="../../../../assets/js/vendor/highlight.js"></script> |
| <script src="../../../../assets/js/vendor/lunr.js"></script> |
| <script src="../../../../assets/js/vendor/search.js" id="search-script" data-base-path="../../../.." data-page-path="/Cassandra/4.1/cassandra/architecture/messaging.html"></script> |
| <script async src="../../../../assets/../search-index.js"></script> |
| <script> |
| jQuery(function(){ |
| var windowW = $(window).width(); |
| $(document) |
| .on('click','.mobile-nav-icon',function(){ |
| $('.main-nav').fadeIn(); |
| }) |
| .on('click','.main-nav',function(){ |
| if(windowW <= 1000){ |
| $(this).fadeOut(); |
| } |
| }) |
| .on('click','#version-toggle',function(){ |
| $(this).toggleClass('active'); |
| $(this).next().fadeToggle(); |
| }) |
| .on('click','#mobile-docs-nav-burger', function(){ |
| $(this).toggleClass('active'); |
| $('.docs-nav').toggleClass('active'); |
| }); |
| var url = window.location.pathname; |
| var isQuickstart = url.includes('quickstart.html'); |
| if(isQuickstart){ |
| var footerCTA = document.getElementById('footer-cta'); |
| footerCTA.innerHTML = 'Get latest updates'; |
| footerCTA.setAttribute('href', '/_/blog.html'); |
| } |
| }); |
| </script> |
| </div> |
| </body> |
| </html> |
