blob: 40c7c14c64176e9ad386f3e13ecc7df2f8ac8dbc [file] [log] [blame]
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1.0">
<title>Find The Misbehaving Nodes | Apache Cassandra Documentation</title>
<link rel="stylesheet" href="../../../../assets/css/site.css">
<link rel="schema.dcterms" href="https://purl.org/dc/terms/">
<meta name="dcterms.subject" content="Cassandra">
<meta name="dcterms.identifier" content="5.0">
<meta name="generator" content="Antora 2.3.4">
<link rel="icon" href="../../../../assets/img/favicon.ico" type="image/x-icon">
<script>
const script = document.createElement("script");
const domain = window.location.hostname;
script.type = "text/javascript";
script.src = "https://plausible.cassandra.apache.org/js/plausible.js";
script.setAttribute("data-domain",domain);
script.setAttribute("defer",'true');
script.setAttribute("async",'true');
document.getElementsByTagName("head")[0].appendChild(script);
</script> </head>
<body class="docs-wrapper article">
<div class="container mx-auto relative">
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.6.0/jquery.min.js"></script>
<meta property="og:type" content="website" />
<meta property="og:url" content="/" />
<meta property="og:site_name" content="Apache Cassandra" />
<header id="top-nav">
<div class="inner relative">
<div class="header-social-icons text-right">
<a href="https://twitter.com/cassandra?lang=en" target="_blank" styles="margin-left: 20px;"><img src="../../../../assets/img/twitter-icon-circle-white.svg" alt="twitter icon" width="24"></a>
<a href="https://www.linkedin.com/company/apache-cassandra/" target="_blank" styles="margin-left: 20px;"><img src="../../../../assets/img/LI-In-Bug.png" alt="linked-in icon" width="24"></a>
<a href="https://www.youtube.com/c/PlanetCassandra" target="_blank" styles="margin-left: 20px;"><img src="../../../../assets/img/youtube-icon.png" alt="youtube icon" width="24"></a>
</div>
<div class="cf">
<div class="logo left"><a href="/"><img src="../../../../assets/img/logo-white-r.png" alt="Cassandra Logo"></a></div>
<div class="mobile-nav-icon right">
<img class="toggle-icon" src="../../../../assets/img/hamburger-nav.svg">
</div>
<ul class="main-nav nav-links right flex flex-vert-center flex-space-between">
<li>
<a class="nav-link hide-mobile">Get Started</a>
<ul class="sub-menu bg-white">
<li class="pa-micro">
<a href="/_/cassandra-basics.html">
<div class="sub-nav-icon">
<img src="../../../../assets/img/sub-menu-basics.png" alt="cassandra basics icon">
</div>
<div class="sub-nav-text teal py-small">
Cassandra Basics
</div>
</a>
</li>
<li class="pa-micro">
<a href="/_/quickstart.html">
<div class="sub-nav-icon">
<img src="../../../../assets/img/sub-menu-rocket.png" alt="cassandra basics icon">
</div>
<div class="sub-nav-text teal py-small">
Quickstart
</div>
</a>
</li>
<li class="pa-micro">
<a href="/_/ecosystem.html">
<div class="sub-nav-icon">
<img src="../../../../assets/img/sub-menu-ecosystem.png" alt="cassandra basics icon">
</div>
<div class="sub-nav-text teal py-small">
Ecosystem
</div>
</a>
</li>
</ul>
</li>
<li><a class="nav-link" href="/doc/latest/">Documentation</a></li>
<li>
<a class="nav-link" href="/_/community.html">Community</a>
<ul class="sub-menu bg-white">
<li class="pa-micro">
<a href="/_/community.html#code-of-conduct">
<div class="sub-nav-icon">
<img src="../../../../assets/img/sub-menu-welcome.png" alt="welcome icon">
</div>
<div class="sub-nav-text teal py-small">
Welcome
</div>
</a>
</li>
<li class="pa-micro hide-mobile">
<a href="/_/community.html#discussions">
<div class="sub-nav-icon">
<img src="../../../../assets/img/sub-menu-discussions.png" alt="discussions icon">
</div>
<div class="sub-nav-text teal py-small">
Discussions
</div>
</a>
</li>
<li class="pa-micro hide-mobile">
<a href="/_/community.html#project-governance">
<div class="sub-nav-icon">
<img src="../../../../assets/img/sub-menu-governance.png" alt="Governance icon">
</div>
<div class="sub-nav-text teal py-small">
Governance
</div>
</a>
</li>
<li class="pa-micro hide-mobile">
<a href="/_/community.html#how-to-contribute">
<div class="sub-nav-icon">
<img src="../../../../assets/img/sub-menu-contribute.png" alt="Contribute icon">
</div>
<div class="sub-nav-text teal py-small">
Contribute
</div>
</a>
</li>
<li class="pa-micro hide-mobile">
<a href="/_/community.html#meet-the-community">
<div class="sub-nav-icon">
<img src="../../../../assets/img/sub-menu-community.png" alt="Meet the Community icon">
</div>
<div class="sub-nav-text teal py-small">
Meet the Community
</div>
</a>
</li>
<li class="pa-micro hide-mobile">
<a href="/_/cassandra-catalyst-program.html">
<div class="sub-nav-icon">
<img src="../../../../assets/img/sub-menu-catalyst.png" alt="Catalyst icon">
</div>
<div class="sub-nav-text teal py-small">
Catalyst Program
</div>
</a>
</li>
<li class="pa-micro hide-mobile">
<a href="/_/events.html">
<div class="sub-nav-icon">
<img src="../../../../assets/img/sub-menu-events.png" alt="Events icon">
</div>
<div class="sub-nav-text teal py-small">
Events
</div>
</a>
</li>
</ul>
</li>
<li>
<a class="nav-link hide-mobile">Learn</a>
<ul class="sub-menu bg-white">
<li class="pa-micro">
<a href="/_/Apache-Cassandra-5.0-Moving-Toward-an-AI-Driven-Future.html">
<div class="sub-nav-icon">
<img src="../../../../assets/img/sub-menu-basics.png" alt="Basics icon">
</div>
<div class="sub-nav-text teal py-small">
Cassandra 5.0
</div>
</a>
</li>
<li class="pa-micro">
<a href="/_/case-studies.html">
<div class="sub-nav-icon">
<img src="../../../../assets/img/sub-menu-case-study.png" alt="Case Studies icon">
</div>
<div class="sub-nav-text teal py-small">
Case Studies
</div>
</a>
</li>
<li class="pa-micro">
<a href="/_/resources.html">
<div class="sub-nav-icon">
<img src="../../../../assets/img/sub-menu-resources.png" alt="Resources icon">
</div>
<div class="sub-nav-text teal py-small">
Resources
</div>
</a>
</li>
<li class="pa-micro">
<a href="/_/blog.html">
<div class="sub-nav-icon">
<img src="../../../../assets/img/sub-menu-blog.png" alt="Blog icon">
</div>
<div class="sub-nav-text teal py-small">
Blog
</div>
</a>
</li>
</ul>
</li>
<li><a class="nav-link btn btn--filled" href="/_/download.html">Download Now</a></li>
</ul>
</div>
</div>
</header>
<div class="hero hero--home grad">
<div class="eye"></div>
<div id="docs-content" class="text-center flex flex-center flex-column relative z2 ma-xlarge">
<h2>Cassandra Documentation</h2>
</div>
</div>
<div class="body px-medium py-medium container">
<div class="docs-nav-bar flex flex-space-between mb-medium">
<div id="mobile-docs-nav-burger" class="hidden">
<svg viewBox="0 0 24 24" width="36" height="36" stroke="#1c81a0" stroke-width="2.5" fill="none" stroke-linecap="round" stroke-linejoin="round" class="css-i6dzq1"><line x1="3" y1="12" x2="21" y2="12"></line><line x1="3" y1="6" x2="21" y2="6"></line><line x1="3" y1="18" x2="21" y2="18"></line></svg>
</div>
<div class="docs-nav-item relative">
<input id="search-input" type="text" placeholder="Search docs">
</div>
<div class="versions-wrapper">
<h4>Version:</h4>
<div class="nav-panel-explore" data-panel="explore">
<div id="version-toggle" class="context">
<span class="version">5.0</span>
</div>
<ul id="versions-list" class="components">
<li class="component">
<ul class="versions">
<li class="version is-latest">
<a href="../../../../_/index.html">master</a>
</li>
</ul>
</li>
<li class="component is-current">
<ul class="versions">
<li class="version">
<a href="../../../trunk/index.html">trunk</a>
</li>
<li class="version is-current">
<a href="../../index.html">5.0</a>
</li>
<li class="version is-latest">
<a href="../../../4.1/index.html">4.1</a>
</li>
<li class="version">
<a href="../../../4.0/index.html">4.0</a>
</li>
<li class="version">
<a href="../../../3.11/index.html">3.11</a>
</li>
</ul>
</li>
</ul>
</div>
</div> </div>
<div class="cf relative">
<nav class="nav docs-nav full-800">
<div class="nav-menu">
<ul class="nav-list">
<li class="nav-item is-active" data-depth="0">
<ul class="nav-list">
<li class="nav-item" data-depth="1">
<span class="nav-line">
<button class="nav-toggle"></button>
<a class="nav-link" href="../../index.html">Main</a>
</span>
<ul class="nav-list">
<li class="nav-item" data-depth="2">
<span class="nav-line">
<a class="nav-link" href="../../../../_/glossary.html">Glossary</a>
</span>
</li>
<li class="nav-item" data-depth="2">
<span class="nav-line">
<a class="nav-link" href="../../../../_/bugs.html">How to report bugs</a>
</span>
</li>
<li class="nav-item" data-depth="2">
<span class="nav-line">
<a class="nav-link" href="../../../../_/contactus.html">Contact us</a>
</span>
</li>
<li class="nav-item" data-depth="2">
<span class="nav-line">
<button class="nav-toggle"></button>
<a class="nav-link" href="../../../../_/development/index.html">Development</a>
</span>
<ul class="nav-list">
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../../../../_/development/gettingstarted.html">Getting started</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../../../../_/development/ide.html">Building and IDE integration</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../../../../_/development/testing.html">Testing</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../../../../_/development/patches.html">Contributing code changes</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../../../../_/development/code_style.html">Code style</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../../../../_/development/how_to_review.html">Review checklist</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../../../../_/development/how_to_commit.html">How to commit</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../../../../_/development/documentation.html">Working on documentation</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../../../../_/development/ci.html">Jenkins CI environment</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../../../../_/development/dependencies.html">Dependency management</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../../../../_/development/release_process.html">Release process</a>
</span>
</li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
<li class="nav-item is-active" data-depth="0">
<ul class="nav-list">
<li class="nav-item is-current-path is-active" data-depth="1">
<span class="nav-line">
<button class="nav-toggle"></button>
<span class="nav-text">Cassandra</span>
</span>
<ul class="nav-list">
<li class="nav-item" data-depth="2">
<span class="nav-line">
<a class="nav-link" href="../overview/faq/index.html">FAQ</a>
</span>
</li>
<li class="nav-item" data-depth="2">
<span class="nav-line">
<button class="nav-toggle"></button>
<a class="nav-link" href="../getting-started/index.html">Getting Started</a>
</span>
<ul class="nav-list">
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../getting-started/cassandra-quickstart.html">Cassandra Quickstart</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../getting-started/sai-quickstart.html">SAI Quickstart</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../getting-started/vector-search-quickstart.html">Vector Search Quickstart</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../installing/installing.html">Installing Cassandra</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../getting-started/configuring.html">Configuring Cassandra</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../getting-started/querying.html">Inserting and querying</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../getting-started/drivers.html">Client drivers</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../getting-started/production.html">Production recommendations</a>
</span>
</li>
</ul>
</li>
<li class="nav-item" data-depth="2">
<span class="nav-line">
<button class="nav-toggle"></button>
<a class="nav-link" href="../new/index.html">What&#8217;s new</a>
</span>
<ul class="nav-list">
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../reference/java17.html">Support for Java</a>
</span>
</li>
</ul>
</li>
<li class="nav-item" data-depth="2">
<span class="nav-line">
<button class="nav-toggle"></button>
<a class="nav-link" href="../architecture/index.html">Architecture</a>
</span>
<ul class="nav-list">
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../architecture/overview.html">Overview</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../architecture/dynamo.html">Dynamo</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../architecture/storage-engine.html">Storage Engine</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../architecture/guarantees.html">Guarantees</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../architecture/messaging.html">Improved Internode Messaging</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../architecture/streaming.html">Improved Streaming</a>
</span>
</li>
</ul>
</li>
<li class="nav-item" data-depth="2">
<span class="nav-line">
<button class="nav-toggle"></button>
<a class="nav-link" href="../developing/data-modeling/index.html">Data Modeling</a>
</span>
<ul class="nav-list">
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../developing/data-modeling/intro.html">Introduction</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../developing/data-modeling/data-modeling_conceptual.html">Conceptual data modeling</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../developing/data-modeling/data-modeling_rdbms.html">RDBMS design</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../developing/data-modeling/data-modeling_queries.html">Defining application queries</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../developing/data-modeling/data-modeling_logical.html">Logical data modeling</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../developing/data-modeling/data-modeling_physical.html">Physical data modeling</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../developing/data-modeling/data-modeling_refining.html">Evaluating and refining data models</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../developing/data-modeling/data-modeling_schema.html">Defining database schema</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../developing/data-modeling/data-modeling_tools.html">Cassandra data modeling tools</a>
</span>
</li>
</ul>
</li>
<li class="nav-item" data-depth="2">
<span class="nav-line">
<button class="nav-toggle"></button>
<a class="nav-link" href="../developing/cql/index.html">Cassandra Query Language (CQL)</a>
</span>
<ul class="nav-list">
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../developing/cql/definitions.html">Definitions</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../developing/cql/types.html">Data types</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../developing/cql/ddl.html">Data definition (DDL)</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../developing/cql/dml.html">Data manipulation (DML)</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../developing/cql/dynamic-data-masking.html">Dynamic Data Masking (DDM)</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../developing/cql/operators.html">Operators</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<button class="nav-toggle"></button>
<a class="nav-link" href="../developing/cql/indexing/indexing-concepts.html">Indexing concepts</a>
</span>
<ul class="nav-list">
<li class="nav-item" data-depth="4">
<span class="nav-line">
<button class="nav-toggle"></button>
<a class="nav-link" href="../developing/cql/indexing/sai/sai-overview.html">SAI Overview</a>
</span>
<ul class="nav-list">
<li class="nav-item" data-depth="5">
<span class="nav-line">
<a class="nav-link" href="../developing/cql/indexing/sai/sai-concepts.html">Concepts</a>
</span>
</li>
<li class="nav-item" data-depth="5">
<span class="nav-line">
<a class="nav-link" href="../getting-started/sai-quickstart.html">SAI Quickstart</a>
</span>
</li>
<li class="nav-item" data-depth="5">
<span class="nav-line">
<a class="nav-link" href="../developing/cql/indexing/sai/sai-faq.html">SAI FAQ</a>
</span>
</li>
<li class="nav-item" data-depth="5">
<span class="nav-line">
<a class="nav-link" href="../developing/cql/indexing/sai/sai-working-with.html">Working with SAI</a>
</span>
</li>
<li class="nav-item" data-depth="5">
<span class="nav-line">
<a class="nav-link" href="../developing/cql/indexing/sai/operations/sai-operations.html">SAI operations</a>
</span>
</li>
</ul>
</li>
<li class="nav-item" data-depth="4">
<span class="nav-line">
<button class="nav-toggle"></button>
<a class="nav-link" href="../developing/cql/indexing/2i/2i-overview.html">Secondary indexes (2i) overview</a>
</span>
<ul class="nav-list">
<li class="nav-item" data-depth="5">
<span class="nav-line">
<a class="nav-link" href="../developing/cql/indexing/2i/2i-concepts.html">Concepts</a>
</span>
</li>
<li class="nav-item" data-depth="5">
<span class="nav-line">
<a class="nav-link" href="../developing/cql/indexing/2i/2i-working-with.html">Working with 2i</a>
</span>
</li>
<li class="nav-item" data-depth="5">
<span class="nav-line">
<a class="nav-link" href="../developing/cql/indexing/2i/operations/2i-build.html">Rebuild 2i</a>
</span>
</li>
</ul>
</li>
</ul>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../developing/cql/mvs.html">Materialized views</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../developing/cql/functions.html">Functions</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../developing/cql/json.html">JSON</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../developing/cql/security.html">Security</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../developing/cql/triggers.html">Triggers</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../developing/cql/appendices.html">Appendices</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../developing/cql/changes.html">Changes</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../developing/cql/SASI.html">SASI</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../developing/cql/cql_singlefile.html">Single file of CQL information</a>
</span>
</li>
</ul>
</li>
<li class="nav-item" data-depth="2">
<span class="nav-line">
<button class="nav-toggle"></button>
<a class="nav-link" href="../vector-search/overview.html">Vector Search overview</a>
</span>
<ul class="nav-list">
<li class="nav-item" data-depth="3">
<span class="nav-line">
<button class="nav-toggle"></button>
<a class="nav-link" href="../vector-search/concepts.html">Concepts</a>
</span>
<ul class="nav-list">
<li class="nav-item" data-depth="4">
<span class="nav-line">
<a class="nav-link" href="../vector-search/data-modeling.html">Data Modeling</a>
</span>
</li>
</ul>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../getting-started/vector-search-quickstart.html">Vector Search Quickstart</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../vector-search/vector-search-working-with.html">Working with Vector Search</a>
</span>
</li>
</ul>
</li>
<li class="nav-item" data-depth="2">
<span class="nav-line">
<button class="nav-toggle"></button>
<a class="nav-link" href="../managing/index.html">Managing</a>
</span>
<ul class="nav-list">
<li class="nav-item" data-depth="3">
<span class="nav-line">
<button class="nav-toggle"></button>
<a class="nav-link" href="../managing/configuration/index.html">Configuring</a>
</span>
<ul class="nav-list">
<li class="nav-item" data-depth="4">
<span class="nav-line">
<a class="nav-link" href="../managing/configuration/cass_yaml_file.html">cassandra.yaml</a>
</span>
</li>
<li class="nav-item" data-depth="4">
<span class="nav-line">
<a class="nav-link" href="../managing/configuration/cass_rackdc_file.html">cassandra-rackdc.properties</a>
</span>
</li>
<li class="nav-item" data-depth="4">
<span class="nav-line">
<a class="nav-link" href="../managing/configuration/cass_env_sh_file.html">cassandra-env.sh</a>
</span>
</li>
<li class="nav-item" data-depth="4">
<span class="nav-line">
<a class="nav-link" href="../managing/configuration/cass_topo_file.html">cassandra-topologies.properties</a>
</span>
</li>
<li class="nav-item" data-depth="4">
<span class="nav-line">
<a class="nav-link" href="../managing/configuration/cass_cl_archive_file.html">commitlog-archiving.properties</a>
</span>
</li>
<li class="nav-item" data-depth="4">
<span class="nav-line">
<a class="nav-link" href="../managing/configuration/cass_logback_xml_file.html">logback.xml</a>
</span>
</li>
<li class="nav-item" data-depth="4">
<span class="nav-line">
<a class="nav-link" href="../managing/configuration/cass_jvm_options_file.html">jvm-* files</a>
</span>
</li>
<li class="nav-item" data-depth="4">
<span class="nav-line">
<a class="nav-link" href="../managing/configuration/configuration.html">Liberating cassandra.yaml Parameters' Names from Their Units</a>
</span>
</li>
</ul>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<button class="nav-toggle"></button>
<a class="nav-link" href="../managing/operating/index.html">Operating</a>
</span>
<ul class="nav-list">
<li class="nav-item" data-depth="4">
<span class="nav-line">
<a class="nav-link" href="../managing/operating/backups.html">Backups</a>
</span>
</li>
<li class="nav-item" data-depth="4">
<span class="nav-line">
<a class="nav-link" href="../managing/operating/bloom_filters.html">Bloom filters</a>
</span>
</li>
<li class="nav-item" data-depth="4">
<span class="nav-line">
<a class="nav-link" href="../managing/operating/bulk_loading.html">Bulk loading</a>
</span>
</li>
<li class="nav-item" data-depth="4">
<span class="nav-line">
<a class="nav-link" href="../managing/operating/cdc.html">Change Data Capture (CDC)</a>
</span>
</li>
<li class="nav-item" data-depth="4">
<span class="nav-line">
<a class="nav-link" href="../managing/operating/compaction/index.html">Compaction</a>
</span>
</li>
<li class="nav-item" data-depth="4">
<span class="nav-line">
<a class="nav-link" href="../managing/operating/compression.html">Compression</a>
</span>
</li>
<li class="nav-item" data-depth="4">
<span class="nav-line">
<a class="nav-link" href="../managing/operating/hardware.html">Hardware</a>
</span>
</li>
<li class="nav-item" data-depth="4">
<span class="nav-line">
<a class="nav-link" href="../managing/operating/hints.html">Hints</a>
</span>
</li>
<li class="nav-item" data-depth="4">
<span class="nav-line">
<button class="nav-toggle"></button>
<a class="nav-link" href="../managing/operating/logging.html">Logging</a>
</span>
<ul class="nav-list">
<li class="nav-item" data-depth="5">
<span class="nav-line">
<a class="nav-link" href="../managing/operating/auditlogging.html">Audit logging</a>
</span>
</li>
<li class="nav-item" data-depth="5">
<span class="nav-line">
<a class="nav-link" href="../managing/operating/audit_logging.html">Audit logging 2</a>
</span>
</li>
<li class="nav-item" data-depth="5">
<span class="nav-line">
<a class="nav-link" href="../managing/operating/fqllogging.html">Full query logging</a>
</span>
</li>
</ul>
</li>
<li class="nav-item" data-depth="4">
<span class="nav-line">
<a class="nav-link" href="../managing/operating/metrics.html">Monitoring metrics</a>
</span>
</li>
<li class="nav-item" data-depth="4">
<span class="nav-line">
<a class="nav-link" href="../managing/operating/repair.html">Repair</a>
</span>
</li>
<li class="nav-item" data-depth="4">
<span class="nav-line">
<a class="nav-link" href="../managing/operating/read_repair.html">Read repair</a>
</span>
</li>
<li class="nav-item" data-depth="4">
<span class="nav-line">
<a class="nav-link" href="../managing/operating/security.html">Security</a>
</span>
</li>
<li class="nav-item" data-depth="4">
<span class="nav-line">
<a class="nav-link" href="../managing/operating/snitch.html">Snitches</a>
</span>
</li>
<li class="nav-item" data-depth="4">
<span class="nav-line">
<a class="nav-link" href="../managing/operating/topo_changes.html">Topology changes</a>
</span>
</li>
<li class="nav-item" data-depth="4">
<span class="nav-line">
<a class="nav-link" href="../managing/operating/transientreplication.html">Transient replication</a>
</span>
</li>
<li class="nav-item" data-depth="4">
<span class="nav-line">
<a class="nav-link" href="../managing/operating/virtualtables.html">Virtual tables</a>
</span>
</li>
</ul>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<button class="nav-toggle"></button>
<a class="nav-link" href="../managing/tools/index.html">Tools</a>
</span>
<ul class="nav-list">
<li class="nav-item" data-depth="4">
<span class="nav-line">
<a class="nav-link" href="../managing/tools/cqlsh.html">cqlsh: the CQL shell</a>
</span>
</li>
<li class="nav-item" data-depth="4">
<span class="nav-line">
<a class="nav-link" href="../managing/tools/nodetool/nodetool.html">nodetool</a>
</span>
</li>
<li class="nav-item" data-depth="4">
<span class="nav-line">
<a class="nav-link" href="../managing/tools/sstable/index.html">SSTable tools</a>
</span>
</li>
<li class="nav-item" data-depth="4">
<span class="nav-line">
<a class="nav-link" href="#cassandra:managing/tools/cassandra_stress.adoc">cassandra-stress</a>
</span>
</li>
</ul>
</li>
</ul>
</li>
<li class="nav-item is-current-path is-active" data-depth="2">
<span class="nav-line">
<button class="nav-toggle"></button>
<a class="nav-link" href="index.html">Troubleshooting</a>
</span>
<ul class="nav-list">
<li class="nav-item is-current-page is-active" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="finding_nodes.html">Finding misbehaving nodes</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="reading_logs.html">Reading Cassandra logs</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="use_nodetool.html">Using nodetool</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="use_tools.html">Using external tools to deep-dive</a>
</span>
</li>
</ul>
</li>
<li class="nav-item" data-depth="2">
<span class="nav-line">
<button class="nav-toggle"></button>
<a class="nav-link" href="../reference/index.html">Reference</a>
</span>
<ul class="nav-list">
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../reference/cql-commands/alter-table.html">ALTER TABLE</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../reference/cql-commands/create-index.html">CREATE INDEX</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../reference/cql-commands/create-custom-index.html">CREATE CUSTOM INDEX</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../reference/cql-commands/create-table.html">CREATE TABLE</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../reference/cql-commands/drop-index.html">DROP INDEX</a>
</span>
</li>
<li class="nav-item" data-depth="3">
<span class="nav-line">
<a class="nav-link" href="../reference/cql-commands/drop-table.html">DROP TABLE</a>
</span>
</li>
</ul>
</li>
<li class="nav-item" data-depth="2">
<span class="nav-line">
<a class="nav-link" href="../integrating/plugins/index.html">Plug-ins</a>
</span>
</li>
</ul>
</li>
</ul>
</li>
</ul>
</div>
</nav>
<aside class="toc sidebar">
<div class="toc-menu"></div>
</aside>
<main class="article default-main full-800" data-ceiling="topbar">
<div class="article-banner">
<p>You are viewing the documentation for a prerelease version.</p>
<a class="btn" href="../../../4.1/cassandra/troubleshooting/finding_nodes.html">View Latest</a>
</div>
<div class="article-header">
<nav class="crumbs" aria-label="breadcrumbs">
<ul>
<li class="crumb">Cassandra</li>
<li class="crumb"><a href="index.html">Troubleshooting</a></li>
<li class="crumb"><a href="finding_nodes.html">Finding misbehaving nodes</a></li>
</ul>
</nav>
<div class="tools" role="navigation">
<ul>
<li class="tool edit"><a href="https://github.com/apache/cassandra/edit/cassandra-5.0/doc/modules/cassandra/pages/troubleshooting/finding_nodes.adoc" title="Edit Page" target="_blank" rel="noopener">Edit</a></li>
</ul>
</div>
</div>
<article class="doc">
<h1 class="page">Find The Misbehaving Nodes</h1>
<div id="preamble">
<div class="sectionbody">
<div class="paragraph">
<p>The first step to troubleshooting a Cassandra issue is to use error
messages, metrics and monitoring information to identify if the issue
lies with the clients or the server and if it does lie with the server
find the problematic nodes in the Cassandra cluster. The goal is to
determine if this is a systemic issue (e.g. a query pattern that affects
the entire cluster) or isolated to a subset of nodes (e.g. neighbors
holding a shared token range or even a single node with bad hardware).</p>
</div>
<div class="paragraph">
<p>There are many sources of information that help determine where the
problem lies. Some of the most common are mentioned below.</p>
</div>
</div>
</div>
<div class="sect1">
<h2 id="client-logs-and-errors"><a class="anchor" href="#client-logs-and-errors"></a>Client Logs and Errors</h2>
<div class="sectionbody">
<div class="paragraph">
<p>Clients of the cluster often leave the best breadcrumbs to follow.
Perhaps client latencies or error rates have increased in a particular
datacenter (likely eliminating other datacenter&#8217;s nodes), or clients are
receiving a particular kind of error code indicating a particular kind
of problem. Troubleshooters can often rule out many failure modes just
by reading the error messages. In fact, many Cassandra error messages
include the last coordinator contacted to help operators find nodes to
start with.</p>
</div>
<div class="paragraph">
<p>Some common errors (likely culprit in parenthesis) assuming the client
has similar error names as the Datastax <code>drivers &lt;client-drivers&gt;</code>:</p>
</div>
<div class="ulist">
<ul>
<li>
<p><code>SyntaxError</code> (<strong>client</strong>). This and other <code>QueryValidationException</code>
indicate that the client sent a malformed request. These are rarely
server issues and usually indicate bad queries.</p>
</li>
<li>
<p><code>UnavailableException</code> (<strong>server</strong>): This means that the Cassandra
coordinator node has rejected the query as it believes that insufficent
replica nodes are available. If many coordinators are throwing this
error it likely means that there really are (typically) multiple nodes
down in the cluster and you can identify them using <code>nodetool status
&lt;nodetool-status&gt;</code> If only a single coordinator is throwing this error
it may mean that node has been partitioned from the rest.</p>
</li>
<li>
<p><code>OperationTimedOutException</code> (<strong>server</strong>): This is the most frequent
timeout message raised when clients set timeouts and means that the
query took longer than the supplied timeout. This is a <em>client side</em>
timeout meaning that it took longer than the client specified timeout.
The error message will include the coordinator node that was last tried
which is usually a good starting point. This error usually indicates
either aggressive client timeout values or latent server
coordinators/replicas.</p>
</li>
<li>
<p><code>ReadTimeoutException</code> or <code>WriteTimeoutException</code> (<strong>server</strong>): These
are raised when clients do not specify lower timeouts and there is a
<em>coordinator</em> timeouts based on the values supplied in the
<code>cassandra.yaml</code> configuration file. They usually indicate a serious
server side problem as the default values are usually multiple seconds.</p>
</li>
</ul>
</div>
</div>
</div>
<div class="sect1">
<h2 id="metrics"><a class="anchor" href="#metrics"></a>Metrics</h2>
<div class="sectionbody">
<div class="paragraph">
<p>If you have Cassandra <a href="../managing/operating/metrics.html" class="page"><code>metrics</code></a> reporting to a
centralized location such as <a href="https://graphiteapp.org/">Graphite</a> or
<a href="https://grafana.com/">Grafana</a> you can typically use those to narrow down
the problem. At this stage narrowing down the issue to a particular
datacenter, rack, or even group of nodes is the main goal. Some helpful
metrics to look at are:</p>
</div>
<div class="sect2">
<h3 id="errors"><a class="anchor" href="#errors"></a>Errors</h3>
<div class="paragraph">
<p>Cassandra refers to internode messaging errors as "drops", and provided
a number of <a href="../managing/operating/metrics.html#droppedmessage-metrics" class="page"><code>Dropped Message Metrics</code></a> to help narrow
down errors. If particular nodes are dropping messages actively, they
are likely related to the issue.</p>
</div>
</div>
<div class="sect2">
<h3 id="latency"><a class="anchor" href="#latency"></a>Latency</h3>
<div class="paragraph">
<p>For timeouts or latency related issues you can start with operating/metrics.adoc#table-metrics[<code>table metrics</code>]
by comparing Coordinator level metrics e.g.
<code>CoordinatorReadLatency</code> or <code>CoordinatorWriteLatency</code> with their
associated replica metrics e.g. <code>ReadLatency</code> or <code>WriteLatency</code>. Issues
usually show up on the <code>99th</code> percentile before they show up on the
<code>50th</code> percentile or the <code>mean</code>. While <code>maximum</code> coordinator latencies
are not typically very helpful due to the exponentially decaying
reservoir used internally to produce metrics, <code>maximum</code> replica
latencies that correlate with increased <code>99th</code> percentiles on
coordinators can help narrow down the problem.</p>
</div>
<div class="paragraph">
<p>There are usually three main possibilities:</p>
</div>
<div class="olist arabic">
<ol class="arabic">
<li>
<p>Coordinator latencies are high on all nodes, but only a few node&#8217;s
local read latencies are high. This points to slow replica nodes and the
coordinator&#8217;s are just side-effects. This usually happens when clients
are not token aware.</p>
</li>
<li>
<p>Coordinator latencies and replica latencies increase at the same time
on the a few nodes. If clients are token aware this is almost always
what happens and points to slow replicas of a subset of token ranges
(only part of the ring).</p>
</li>
<li>
<p>Coordinator and local latencies are high on many nodes. This usually
indicates either a tipping point in the cluster capacity (too many
writes or reads per second), or a new query pattern.</p>
</li>
</ol>
</div>
<div class="paragraph">
<p>It&#8217;s important to remember that depending on the client&#8217;s load balancing
behavior and consistency levels coordinator and replica metrics may or
may not correlate. In particular if you use <code>TokenAware</code> policies the
same node&#8217;s coordinator and replica latencies will often increase
together, but if you just use normal <code>DCAwareRoundRobin</code> coordinator
latencies can increase with unrelated replica node&#8217;s latencies. For
example:</p>
</div>
<div class="ulist">
<ul>
<li>
<p><code>TokenAware</code> + <code>LOCAL_ONE</code>: should always have coordinator and replica
latencies on the same node rise together</p>
</li>
<li>
<p><code>TokenAware</code> + <code>LOCAL_QUORUM</code>: should always have coordinator and
multiple replica latencies rise together in the same datacenter.</p>
</li>
<li>
<p><code>TokenAware</code> + <code>QUORUM</code>: replica latencies in other datacenters can
affect coordinator latencies.</p>
</li>
<li>
<p><code>DCAwareRoundRobin</code> + <code>LOCAL_ONE</code>: coordinator latencies and unrelated
replica node&#8217;s latencies will rise together.</p>
</li>
<li>
<p><code>DCAwareRoundRobin</code> + <code>LOCAL_QUORUM</code>: different coordinator and
replica latencies will rise together with little correlation.</p>
</li>
</ul>
</div>
</div>
<div class="sect2">
<h3 id="query-rates"><a class="anchor" href="#query-rates"></a>Query Rates</h3>
<div class="paragraph">
<p>Sometimes the <a href="../managing/operating/metrics.html#table-metrics" class="page"><code>table metric</code></a> query rate metrics can help narrow
down load issues as "small" increase in coordinator queries per second
(QPS) may correlate with a very large increase in replica level QPS.
This most often happens with <code>BATCH</code> writes, where a client may send a
single <code>BATCH</code> query that might contain 50 statements in it, which if
you have 9 copies (RF=3, three datacenters) means that every coordinator
<code>BATCH</code> write turns into 450 replica writes! This is why keeping
`BATCH&#8217;s to the same partition is so critical, otherwise you can
exhaust significant CPU capacitity with a "single" query.</p>
</div>
</div>
</div>
</div>
<div class="sect1">
<h2 id="next-step-investigate-the-nodes"><a class="anchor" href="#next-step-investigate-the-nodes"></a>Next Step: Investigate the Node(s)</h2>
<div class="sectionbody">
<div class="paragraph">
<p>Once you have narrowed down the problem as much as possible (datacenter,
rack , node), login to one of the nodes using SSH and proceed to debug
using <a href="#reading_logs.adoc" class="page unresolved"><code>logs</code></a>, <a href="#use_nodetooladoc"><code>nodetool</code></a>, and
<a href="#use_tools.adoc" class="page unresolved"><code>os tools</code></a>.
If you are not able to login you may still have access to <code>logs</code> and <code>nodetool</code> remotely.</p>
</div>
</div>
</div>
</article>
</main>
</div>
</div>
<footer class="grad grad--two flex-center pb-xlarge">
<div class="inner text-center z2 relative">
<h2 class="white py-small">Get started with Cassandra, fast.</h2>
<a id="footer-cta" href="/_/quickstart.html" class="btn btn--filled ma-medium">Quickstart Guide</a>
</div>
<div class="inner flex flex-distribute-items mt-xlarge z2 relative">
<div class="col-2">
<div id="footer-logo" class="logo logo--footer mb-medium"><img src="../../../../assets/img/logo-white-r.png" alt="Cassandra Logo"></div>
<p>Apache Cassandra<img src="../../../../assets/img/registered.svg" alt="®" style="width:18px;"> powers mission-critical deployments with improved performance and unparalleled levels of scale in the cloud.</p>
<div class="footer-social-icons">
<a href="https://twitter.com/cassandra?lang=en" target="_blank"><img src="../../../../assets/img/twitter-icon-circle-white.svg" alt="twitter icon" width="24"></a>
<a href="https://www.linkedin.com/company/apache-cassandra/" target="_blank"><img src="../../../../assets/img/LI-In-Bug.png" alt="linked-in icon" width="24"></a>
<a href="https://www.youtube.com/c/PlanetCassandra" target="_blank"><img src="../../../../assets/img/youtube-icon.png" alt="youtube icon" width="24"></a>
</div>
</div>
<div class="col-2 flex flex-center">
<ul class="columns-2">
<li class="mb-small"><a href="/">Home</a></li>
<li class="mb-small"><a href="/_/cassandra-basics.html">Cassandra Basics</a></li>
<li class="mb-small"><a href="/_/quickstart.html">Quickstart</a></li>
<li class="mb-small"><a href="/_/ecosystem.html">Ecosystem</a></li>
<li class="mb-small"><a href="/doc/latest/">Documentation</a></li>
<li class="mb-small"><a href="/_/community.html">Community</a></li>
<li class="mb-small"><a href="/_/case-studies.html">Case Studies</a></li>
<li class="mb-small"><a href="/_/resources.html">Resources</a></li>
<li class="mb-small"><a href="/_/blog.html">Blog</a></li>
</ul>
</div>
</div>
</footer>
<div class="lower-footer bg-white pa-medium">
<div class="flex flex-row flex-vert-center">
<div class="pr-medium"><img src="../../../../assets/img//feather-small.png" alt="ASF" width="20"></div>
<div class="pr-medium"><a href="http://www.apache.org/" target="_blank">Foundation</a></div>
<div class="pr-medium"><a href="https://www.apache.org/events/current-event.html" target="_blank">Events</a></div>
<div class="pr-medium"><a href="https://www.apache.org/licenses/" target="_blank">License</a></div>
<div class="pr-medium"><a href="https://www.apache.org/foundation/thanks" target="_blank">Thanks</a></div>
<div class="pr-medium"><a href="https://www.apache.org/security" target="_blank">Security</a></div>
<div class="pr-medium"><a href="https://privacy.apache.org/policies/privacy-policy-public.html" target="_blank">Privacy</a></div>
<div class="pr-medium"><a href="https://www.apache.org/foundation/sponsorship" target="_blank">Sponsorship</a></div>
</div>
<p class="my-medium">© 2009-<script>document.write(new Date().getFullYear())</script> <a href="https://apache.org" target="_blank">The Apache Software Foundation</a> under the terms of the Apache License 2.0. Apache, the Apache feather logo, Apache Cassandra, Cassandra, and the Cassandra logo, are either registered trademarks or trademarks of The Apache Software Foundation.</p>
</div>
<div id="fade" class="hidden"></div>
<div id="modal" class="hidden">
<div id="close-modal" class="cursor-pointer"><svg viewBox="0 0 24 24" width="24" height="24" stroke="currentColor" stroke-width="2" fill="none" stroke-linecap="round" stroke-linejoin="round" class="css-i6dzq1"><line x1="18" y1="6" x2="6" y2="18"></line><line x1="6" y1="6" x2="18" y2="18"></line></svg></div>
<div id="mod-content" class="vid-mod-content resp-container"></div>
</div>
<script src="../../../../assets/js/site.js"></script>
<script async src="../../../../assets/js/vendor/highlight.js"></script>
<script src="../../../../assets/js/vendor/lunr.js"></script>
<script src="../../../../assets/js/vendor/search.js" id="search-script" data-base-path="../../../.." data-page-path="/Cassandra/5.0/cassandra/troubleshooting/finding_nodes.html"></script>
<script async src="../../../../assets/../search-index.js"></script>
<script>
jQuery(function(){
var windowW = $(window).width();
$(document)
.on('click','.mobile-nav-icon',function(){
$('.main-nav').fadeIn();
})
.on('click','.main-nav',function(){
if(windowW <= 1000){
$(this).fadeOut();
}
})
.on('click','#version-toggle',function(){
$(this).toggleClass('active');
$(this).next().fadeToggle();
})
.on('click','#mobile-docs-nav-burger', function(){
$(this).toggleClass('active');
$('.docs-nav').toggleClass('active');
});
var url = window.location.pathname;
var isQuickstart = url.includes('quickstart.html');
if(isQuickstart){
var footerCTA = document.getElementById('footer-cta');
footerCTA.innerHTML = 'Get latest updates';
footerCTA.setAttribute('href', '/_/blog.html');
}
});
</script>
</div>
</body>
</html>