| // Licensed to the Apache Software Foundation (ASF) under one |
| // or more contributor license agreements. See the NOTICE file |
| // distributed with this work for additional information |
| // regarding copyright ownership. The ASF licenses this file |
| // to you under the Apache License, Version 2.0 (the |
| // "License"); you may not use this file except in compliance |
| // with the License. You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, |
| // software distributed under the License is distributed on an |
| // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| // KIND, either express or implied. See the License for the |
| // specific language governing permissions and limitations |
| // under the License. |
| |
| #include "kudu/tools/ksck.h" |
| |
| #include <algorithm> |
| #include <atomic> |
| #include <cstddef> |
| #include <functional> |
| #include <iomanip> |
| #include <iostream> |
| #include <iterator> |
| #include <map> |
| #include <mutex> |
| #include <set> |
| #include <type_traits> |
| #include <vector> |
| |
| #include <gflags/gflags.h> |
| #include <glog/logging.h> |
| #include <google/protobuf/stubs/port.h> |
| |
| #include "kudu/common/partition.h" |
| #include "kudu/consensus/quorum_util.h" |
| #include "kudu/gutil/basictypes.h" |
| #include "kudu/gutil/map-util.h" |
| #include "kudu/gutil/port.h" |
| #include "kudu/gutil/strings/join.h" |
| #include "kudu/gutil/strings/split.h" |
| #include "kudu/gutil/strings/substitute.h" |
| #include "kudu/tablet/tablet.pb.h" |
| #include "kudu/tools/color.h" |
| #include "kudu/tools/ksck_checksum.h" |
| #include "kudu/util/locks.h" |
| #include "kudu/util/monotime.h" |
| #include "kudu/util/string_case.h" |
| |
| #define PUSH_PREPEND_NOT_OK(s, statuses, msg) do { \ |
| ::kudu::Status _s = (s); \ |
| if (PREDICT_FALSE(!_s.ok())) { \ |
| (statuses).push_back(_s.CloneAndPrepend(msg)); \ |
| } \ |
| } while (0) |
| |
| #define STR_FLAGS_CATEGORY_TIME_SOURCE "time_source" |
| #define STR_FLAGS_CATEGORY_UNUSUAL "unusual" |
| |
| DEFINE_bool(checksum_scan, false, |
| "Perform a checksum scan on data in the cluster."); |
| DEFINE_int32(fetch_info_concurrency, 20, |
| "Number of threads to fetch info concurrently."); |
| DEFINE_string(flags_categories_to_check, STR_FLAGS_CATEGORY_TIME_SOURCE, |
| "Comma-separated list of flag categories to check for divergence " |
| "across the cluster; default is " |
| STR_FLAGS_CATEGORY_TIME_SOURCE "; available categories are " |
| STR_FLAGS_CATEGORY_TIME_SOURCE ", " |
| STR_FLAGS_CATEGORY_UNUSUAL "."); |
| |
| DEFINE_string(ksck_format, "plain_concise", |
| "Output format for ksck. Available options are 'plain_concise', " |
| "'plain_full', 'json_pretty', and 'json_compact'.\n" |
| "'plain_concise' format is plain text, omitting most information " |
| "about healthy tablets.\n" |
| "'plain_full' is plain text with all results included.\n" |
| "'json_pretty' produces pretty-printed json.\n" |
| "'json_compact' produces json suitable for parsing by other programs.\n" |
| "'json_pretty' and 'json_compact' differ in format, not content."); |
| DEFINE_bool(consensus, true, |
| "Whether to check the consensus state from each tablet against the master."); |
| |
| using kudu::cluster_summary::HealthCheckResult; |
| using kudu::cluster_summary::ConsensusConfigType; |
| using kudu::cluster_summary::ConsensusState; |
| using kudu::cluster_summary::ConsensusStateMap; |
| using kudu::cluster_summary::ReplicaSummary; |
| using kudu::cluster_summary::ServerHealth; |
| using kudu::cluster_summary::ServerHealthSummary; |
| using kudu::cluster_summary::TableSummary; |
| using kudu::cluster_summary::TabletSummary; |
| using kudu::server::GetFlagsResponsePB; |
| |
| using std::atomic; |
| using std::cout; |
| using std::nullopt; |
| using std::optional; |
| using std::ostream; |
| using std::ostringstream; |
| using std::pair; |
| using std::set; |
| using std::shared_ptr; |
| using std::string; |
| using std::vector; |
| using strings::Substitute; |
| |
| namespace kudu { |
| namespace tools { |
| |
| DEFINE_validator(flags_categories_to_check, |
| [](const char* /* flag_name */, const string& value) { |
| vector<string> categories; |
| SplitStringUsing(value, ",", &categories); |
| for (const auto& cat : categories) { |
| if (cat.empty() || StringToFlagsCategory(cat, nullptr).ok()) { |
| continue; |
| } |
| LOG(ERROR) << Substitute("unknown flag category: '$0' " |
| "(expecting comma-separated list built out of " |
| STR_FLAGS_CATEGORY_TIME_SOURCE ", " |
| STR_FLAGS_CATEGORY_UNUSUAL ")", cat); |
| return false; |
| } |
| return true; |
| }); |
| |
| namespace { |
| void BuildConsensusStateForConfigMember(const consensus::ConsensusStatePB& cstate, |
| ConsensusState* ksck_cstate) { |
| CHECK(ksck_cstate); |
| ksck_cstate->term = cstate.current_term(); |
| ksck_cstate->type = cstate.has_pending_config() ? |
| ConsensusConfigType::PENDING : |
| ConsensusConfigType::COMMITTED; |
| const auto& config = cstate.has_pending_config() ? |
| cstate.pending_config() : |
| cstate.committed_config(); |
| if (config.has_opid_index()) { |
| ksck_cstate->opid_index = config.opid_index(); |
| } |
| // Test for emptiness rather than mere presence, since Kudu nodes set |
| // leader_uuid to "" explicitly when they do not know about a leader. |
| if (!cstate.leader_uuid().empty()) { |
| ksck_cstate->leader_uuid = cstate.leader_uuid(); |
| } |
| const auto& peers = config.peers(); |
| for (const auto& pb : peers) { |
| if (pb.member_type() == consensus::RaftPeerPB::NON_VOTER) { |
| InsertOrDie(&ksck_cstate->non_voter_uuids, pb.permanent_uuid()); |
| } else { |
| InsertOrDie(&ksck_cstate->voter_uuids, pb.permanent_uuid()); |
| } |
| } |
| } |
| |
| bool IsNotAuthorizedMethodAccess(const Status& s) { |
| return s.IsRemoteError() && |
| s.ToString().find("Not authorized: unauthorized access to method") != string::npos; |
| } |
| |
| // Return whether the format of the ksck results is non-JSON. |
| bool IsNonJSONFormat() { |
| return iequals(FLAGS_ksck_format, "plain_full") || |
| iequals(FLAGS_ksck_format, "plain_concise"); |
| } |
| |
| } // anonymous namespace |
| |
| tablet::TabletStatePB KsckTabletServer::ReplicaState(const std::string& tablet_id) const { |
| CHECK_EQ(state_, KsckFetchState::FETCHED); |
| if (!ContainsKey(tablet_status_map_, tablet_id)) { |
| return tablet::UNKNOWN; |
| } |
| return tablet_status_map_.at(tablet_id).state(); |
| } |
| |
| std::ostream& operator<<(std::ostream& lhs, KsckFetchState state) { |
| switch (state) { |
| case KsckFetchState::UNINITIALIZED: |
| lhs << "UNINITIALIZED"; |
| break; |
| case KsckFetchState::FETCH_FAILED: |
| lhs << "FETCH_FAILED"; |
| break; |
| case KsckFetchState::FETCHED: |
| lhs << "FETCHED"; |
| break; |
| default: |
| LOG(FATAL) << "unknown KsckFetchState"; |
| } |
| return lhs; |
| } |
| |
| const FlagsFetchFilter& GetFlagsCategoryFilter(FlagsCategory category) { |
| // NOTE: using double braces for std::array aggregate initialization. |
| static const std::array<FlagsFetchFilter, FlagsCategory::MAX + 1> kFilters { { |
| { |
| // FlagsCategory::TIME_SOURCE |
| { "time_source", "builtin_ntp_servers", }, |
| {} |
| }, |
| { |
| // FlagsCategory::UNUSUAL |
| {}, |
| { "experimental", "hidden", "unsafe" } |
| }, |
| } }; |
| DCHECK_GE(category, FlagsCategory::MIN); |
| DCHECK_LE(category, FlagsCategory::MAX); |
| return kFilters[category]; |
| } |
| |
| const char* FlagsCategoryToString(FlagsCategory category) { |
| static constexpr const char* const kCategoryTimeSource = |
| STR_FLAGS_CATEGORY_TIME_SOURCE; |
| static constexpr const char* const kCategoryUnusual = |
| STR_FLAGS_CATEGORY_UNUSUAL; |
| switch (category) { |
| case FlagsCategory::TIME_SOURCE: |
| return kCategoryTimeSource; |
| case FlagsCategory::UNUSUAL: |
| return kCategoryUnusual; |
| } |
| return "unknown"; |
| } |
| |
| Status StringToFlagsCategory(const string& str, FlagsCategory* category) { |
| if (iequals(str, STR_FLAGS_CATEGORY_TIME_SOURCE)) { |
| if (category) { |
| *category = FlagsCategory::TIME_SOURCE; |
| } |
| return Status::OK(); |
| } |
| if (iequals(str, STR_FLAGS_CATEGORY_UNUSUAL)) { |
| if (category) { |
| *category = FlagsCategory::UNUSUAL; |
| } |
| return Status::OK(); |
| } |
| return Status::InvalidArgument(Substitute("$0: unknown flags category", str)); |
| } |
| |
| Status StringToFlagsCategories(const std::string& str, |
| vector<FlagsCategory>* categories) { |
| DCHECK(categories); |
| vector<string> categories_str(strings::Split(str, ",", strings::SkipEmpty())); |
| for (const auto& str : categories_str) { |
| FlagsCategory cat; |
| RETURN_NOT_OK(StringToFlagsCategory(str, &cat)); |
| categories->push_back(cat); |
| } |
| std::sort(categories->begin(), categories->end()); |
| categories->erase(std::unique(categories->begin(), categories->end()), |
| categories->end()); |
| return Status::OK(); |
| } |
| |
| Ksck::Ksck(shared_ptr<KsckCluster> cluster, ostream* out) |
| : cluster_(std::move(cluster)), |
| out_(out == nullptr ? &std::cout : out) { |
| CHECK_OK(ThreadPoolBuilder("ksck-fetch") |
| .set_max_threads(FLAGS_fetch_info_concurrency) |
| .set_idle_timeout(MonoDelta::FromMilliseconds(10)) |
| .Build(&pool_)); |
| } |
| |
| Status Ksck::CheckMasterHealth() { |
| const auto num_masters = cluster_->masters().size(); |
| if (num_masters == 0) { |
| return Status::NotFound("no masters found"); |
| } |
| |
| atomic<size_t> bad_masters(0); |
| atomic<size_t> unauthorized_masters(0); |
| |
| vector<ServerHealthSummary> master_summaries; |
| simple_spinlock master_summaries_lock; |
| |
| vector<FlagsCategory> flags_categories_to_fetch = { FlagsCategory::UNUSUAL }; |
| RETURN_NOT_OK(StringToFlagsCategories(FLAGS_flags_categories_to_check, |
| &flags_categories_to_fetch)); |
| for (const auto& master : cluster_->masters()) { |
| RETURN_NOT_OK(pool_->Submit([&]() { |
| ServerHealthSummary sh; |
| Status s = master->FetchInfo().AndThen([&]() { |
| return master->FetchConsensusState(); |
| }); |
| sh.uuid = master->uuid(); |
| sh.address = master->address(); |
| sh.version = master->version(); |
| sh.status = s; |
| if (!s.ok()) { |
| if (IsNotAuthorizedMethodAccess(s)) { |
| sh.health = ServerHealth::UNAUTHORIZED; |
| ++unauthorized_masters; |
| } else { |
| sh.health = ServerHealth::UNAVAILABLE; |
| } |
| ++bad_masters; |
| } |
| |
| { |
| std::lock_guard<simple_spinlock> lock(master_summaries_lock); |
| master_summaries.emplace_back(std::move(sh)); |
| } |
| |
| // Fetch the flags information in every requested category. |
| // Flag retrieval is not supported by older versions; failure is tracked |
| // in CheckTabletServer{Unusual,Diverged}Flags(). |
| ignore_result(master->FetchFlags(flags_categories_to_fetch)); |
| })); |
| } |
| pool_->Wait(); |
| |
| results_.cluster_status.master_summaries.swap(master_summaries); |
| |
| // Return a NotAuthorized status if any master has auth errors, since this |
| // indicates ksck may not be able to gather full and accurate info. |
| if (unauthorized_masters > 0) { |
| return Status::NotAuthorized( |
| Substitute("failed to gather info from $0 of $1 " |
| "masters due to lack of admin privileges", |
| unauthorized_masters.load(), num_masters)); |
| } |
| if (bad_masters > 0) { |
| return Status::NetworkError( |
| Substitute("failed to gather info from all masters: $0 of $1 had errors", |
| bad_masters.load(), num_masters)); |
| } |
| return Status::OK(); |
| } |
| |
| Status Ksck::CheckMasterConsensus() { |
| // Reset this instance's view of master consensus conflict, in case this |
| // instance is being used to repeatedly check for master consensus conflict. |
| results_.cluster_status.master_consensus_conflict = false; |
| if (!FLAGS_consensus) { |
| return Status::OK(); |
| } |
| ConsensusStateMap master_cstates; |
| for (const KsckCluster::MasterList::value_type& master : cluster_->masters()) { |
| if (master->cstate()) { |
| ConsensusState ksck_cstate; |
| BuildConsensusStateForConfigMember(*master->cstate(), &ksck_cstate); |
| InsertOrDie(&master_cstates, master->uuid(), ksck_cstate); |
| } else { |
| results_.cluster_status.master_consensus_conflict = true; |
| } |
| } |
| if (master_cstates.empty()) { |
| return Status::NotFound("no master consensus state available"); |
| } |
| // There's no "reference" cstate for masters, so pick an arbitrary master |
| // cstate to compare with. |
| const ConsensusState& base = master_cstates.begin()->second; |
| for (const auto& entry : master_cstates) { |
| if (!base.Matches(entry.second)) { |
| results_.cluster_status.master_consensus_conflict = true; |
| break; |
| } |
| } |
| results_.cluster_status.master_consensus_state_map.swap(master_cstates); |
| vector<string> uuids; |
| std::transform(cluster_->masters().begin(), cluster_->masters().end(), |
| std::back_inserter(uuids), |
| [](const shared_ptr<KsckMaster>& master) { return master->uuid(); }); |
| results_.cluster_status.master_uuids.swap(uuids); |
| |
| if (results_.cluster_status.master_consensus_conflict) { |
| return Status::Corruption("there are master consensus conflicts"); |
| } |
| return Status::OK(); |
| } |
| |
| void Ksck::AddFlagsToFlagMaps(const GetFlagsResponsePB& flags, |
| const string& server_address, |
| KsckFlagToServersMap* flags_to_servers_map, |
| KsckFlagTagsMap* flag_tags_map) { |
| DCHECK(flags_to_servers_map); |
| for (const auto& f : flags.flags()) { |
| const pair<string, string> key(f.name(), f.value()); |
| if (!InsertIfNotPresent(flags_to_servers_map, key, { server_address })) { |
| FindOrDieNoPrint(*flags_to_servers_map, key).push_back(server_address); |
| } |
| if (flag_tags_map != nullptr) { |
| InsertIfNotPresent(flag_tags_map, f.name(), JoinStrings(f.tags(), ",")); |
| } |
| } |
| } |
| |
| Status Ksck::CheckMasterUnusualFlags() { |
| size_t bad_servers = 0; |
| for (const auto& master : cluster_->masters()) { |
| const auto& unusual_flags = master->flags(FlagsCategory::UNUSUAL); |
| if (!unusual_flags) { |
| ++bad_servers; |
| continue; |
| } |
| AddFlagsToFlagMaps(*unusual_flags, |
| master->address(), |
| &results_.master_unusual_flag_to_servers_map, |
| &results_.master_unusual_flag_tags_map); |
| } |
| |
| if (!results_.master_unusual_flag_to_servers_map.empty()) { |
| results_.warning_messages.push_back(Status::ConfigurationError( |
| "Some masters have unsafe, experimental, or hidden flags set")); |
| } |
| |
| if (bad_servers > 0) { |
| return Status::Incomplete(Substitute( |
| "$0 of $1 masters were not available to retrieve unusual flags", |
| bad_servers, cluster_->masters().size())); |
| } |
| return Status::OK(); |
| } |
| |
| Status Ksck::CheckMasterDivergedFlags() { |
| vector<FlagsCategory> flags_categories; |
| RETURN_NOT_OK(StringToFlagsCategories(FLAGS_flags_categories_to_check, |
| &flags_categories)); |
| for (const auto cat : flags_categories) { |
| KsckFlagToServersMap servers_by_flag; |
| size_t bad_servers = 0; |
| for (const auto& master : cluster_->masters()) { |
| const auto& flags = master->flags(cat); |
| if (!flags) { |
| ++bad_servers; |
| continue; |
| } |
| AddFlagsToFlagMaps(*flags, master->address(), &servers_by_flag); |
| AddFlagsToFlagMaps(*flags, |
| master->address(), |
| &results_.master_checked_flag_to_servers_map); |
| } |
| |
| for (const auto& e : servers_by_flag) { |
| if (e.second.size() + bad_servers == cluster_->masters().size()) { |
| continue; |
| } |
| results_.warning_messages.push_back(Status::ConfigurationError( |
| Substitute("Different masters have different settings for same " |
| "flags of checked category '$0'", |
| FlagsCategoryToString(cat)))); |
| break; |
| } |
| if (bad_servers > 0) { |
| return Status::Incomplete(Substitute( |
| "$0 of $1 masters were not available to retrieve $2 category flags", |
| bad_servers, cluster_->masters().size(), FlagsCategoryToString(cat))); |
| } |
| } |
| return Status::OK(); |
| } |
| |
| Status Ksck::CheckClusterRunning() { |
| VLOG(1) << "Connecting to the leader master"; |
| return cluster_->Connect(); |
| } |
| |
| Status Ksck::FetchTableAndTabletInfo() { |
| return cluster_->FetchTableAndTabletInfo(); |
| } |
| |
| Status Ksck::FetchInfoFromTabletServers() { |
| const auto servers_count = cluster_->tablet_servers().size(); |
| VLOG(1) << Substitute("List of $0 tablet servers retrieved", servers_count); |
| |
| if (servers_count == 0) { |
| return Status::OK(); |
| } |
| |
| atomic<size_t> bad_servers(0); |
| atomic<size_t> unauthorized_servers(0); |
| VLOG(1) << "Fetching info from all " << servers_count << " tablet servers"; |
| |
| vector<ServerHealthSummary> tablet_server_summaries; |
| simple_spinlock tablet_server_summaries_lock; |
| |
| vector<FlagsCategory> flags_categories_to_fetch = { FlagsCategory::UNUSUAL }; |
| RETURN_NOT_OK(StringToFlagsCategories(FLAGS_flags_categories_to_check, |
| &flags_categories_to_fetch)); |
| for (const auto& entry : cluster_->tablet_servers()) { |
| const auto& ts = entry.second; |
| RETURN_NOT_OK(pool_->Submit([&]() { |
| VLOG(1) << "Going to connect to tablet server: " << ts->uuid(); |
| ServerHealth health; |
| Status s = ts->FetchInfo(&health).AndThen([&ts, &health]() { |
| if (FLAGS_consensus) { |
| return ts->FetchConsensusState(&health); |
| } |
| return Status::OK(); |
| }); |
| ServerHealthSummary summary; |
| summary.uuid = ts->uuid(); |
| summary.address = ts->address(); |
| summary.ts_location = ts->location(); |
| summary.version = ts->version(); |
| summary.quiescing_info = ts->quiescing_info(); |
| summary.status = s; |
| if (!s.ok()) { |
| if (IsNotAuthorizedMethodAccess(s)) { |
| health = ServerHealth::UNAUTHORIZED; |
| ++unauthorized_servers; |
| } |
| ++bad_servers; |
| } |
| summary.health = health; |
| |
| { |
| std::lock_guard<simple_spinlock> lock(tablet_server_summaries_lock); |
| tablet_server_summaries.push_back(std::move(summary)); |
| } |
| |
| // Fetch the flags information in every requested category. |
| // Flag retrieval is not supported by older versions; failure is tracked |
| // in CheckTabletServer{Unusual,Diverged}Flags(). |
| ignore_result(ts->FetchFlags(flags_categories_to_fetch)); |
| })); |
| } |
| pool_->Wait(); |
| |
| results_.cluster_status.tserver_summaries.swap(tablet_server_summaries); |
| |
| // Return a NotAuthorized status if any tablet server has auth errors, since |
| // this indicates ksck may not be able to gather full and accurate info. |
| if (unauthorized_servers > 0) { |
| return Status::NotAuthorized( |
| Substitute("failed to gather info from $0 of $1 tablet servers due " |
| "to lack of admin privileges", |
| unauthorized_servers.load(), servers_count)); |
| } |
| if (bad_servers > 0) { |
| return Status::NetworkError( |
| Substitute("failed to gather info for all tablet servers: $0 of $1 had errors", |
| bad_servers.load(), servers_count)); |
| } |
| return Status::OK(); |
| } |
| |
| void Ksck::set_print_sections(const std::vector<std::string>& sections) { |
| print_sections_flags_ = PrintSections::NONE; |
| for (const auto& section : sections) { |
| std::string section_upper; |
| ToUpperCase(section, §ion_upper); |
| if (section_upper == "*") { |
| print_sections_flags_ = PrintSections::ALL_SECTIONS; |
| break; |
| } |
| if (section_upper == "MASTER_SUMMARIES") { |
| print_sections_flags_ |= PrintSections::MASTER_SUMMARIES; |
| } |
| if (section_upper == "TSERVER_STATES") { |
| print_sections_flags_ |= PrintSections::TSERVER_STATES; |
| } |
| if (section_upper == "TSERVER_SUMMARIES") { |
| print_sections_flags_ |= PrintSections::TSERVER_SUMMARIES; |
| } |
| if (section_upper == "VERSION_SUMMARIES") { |
| print_sections_flags_ |= PrintSections::VERSION_SUMMARIES; |
| } |
| if (section_upper == "TABLET_SUMMARIES") { |
| print_sections_flags_ |= PrintSections::TABLET_SUMMARIES; |
| } |
| if (section_upper == "TABLE_SUMMARIES") { |
| print_sections_flags_ |= PrintSections::TABLE_SUMMARIES; |
| } |
| if (section_upper == "SYSTEM_TABLE_SUMMARIES") { |
| print_sections_flags_ |= PrintSections::SYSTEM_TABLE_SUMMARIES; |
| } |
| if (section_upper == "CHECKSUM_RESULTS") { |
| print_sections_flags_ |= PrintSections::CHECKSUM_RESULTS; |
| } |
| if (section_upper == "TOTAL_COUNT") { |
| print_sections_flags_ |= PrintSections::TOTAL_COUNT; |
| } |
| } |
| } |
| |
| Status Ksck::Run() { |
| PUSH_PREPEND_NOT_OK(CheckMasterHealth(), results_.error_messages, |
| "error fetching info from masters"); |
| PUSH_PREPEND_NOT_OK(CheckMasterConsensus(), results_.error_messages, |
| "master consensus error"); |
| PUSH_PREPEND_NOT_OK(CheckMasterUnusualFlags(), results_.warning_messages, |
| "master unusual flags check error"); |
| PUSH_PREPEND_NOT_OK(CheckMasterDivergedFlags(), results_.warning_messages, |
| "master diverged flags check error"); |
| |
| // CheckClusterRunning and FetchTableAndTabletInfo must succeed for |
| // subsequent checks to be runnable. |
| const char* const liveness_prefix = "leader master liveness check error"; |
| Status s = CheckClusterRunning(); |
| PUSH_PREPEND_NOT_OK(s, results_.error_messages, liveness_prefix); |
| RETURN_NOT_OK_PREPEND(s, liveness_prefix); |
| const char* const fetch_prefix = "error fetching the cluster metadata " |
| "from the leader master"; |
| s = FetchTableAndTabletInfo(); |
| PUSH_PREPEND_NOT_OK(s, results_.error_messages, fetch_prefix); |
| RETURN_NOT_OK_PREPEND(s, fetch_prefix); |
| |
| // In getting table and tablet info, we should have also received info about |
| // the tablet servers, including any special states they might be in. |
| results_.ts_states = cluster_->ts_states(); |
| |
| PUSH_PREPEND_NOT_OK(FetchInfoFromTabletServers(), results_.error_messages, |
| "error fetching info from tablet servers"); |
| PUSH_PREPEND_NOT_OK(CheckTabletServerUnusualFlags(), results_.warning_messages, |
| "tserver unusual flags check error"); |
| PUSH_PREPEND_NOT_OK(CheckTabletServerDivergedFlags(), results_.warning_messages, |
| "tserver diverged flags check error"); |
| PUSH_PREPEND_NOT_OK(CheckServerVersions(), results_.warning_messages, |
| "version check error"); |
| |
| PUSH_PREPEND_NOT_OK(CheckDivergedFlags(), results_.warning_messages, |
| "diverged flags (both masters and tservers) check error"); |
| |
| PUSH_PREPEND_NOT_OK(CheckTablesConsistency(), results_.error_messages, |
| "table consistency check error"); |
| |
| if (FLAGS_checksum_scan) { |
| // Copy the filters because they are passed by-value. |
| auto table_filters_for_checksum_opts = cluster_->table_filters(); |
| auto tablet_id_filters_for_checksum_opts = cluster_->tablet_id_filters(); |
| PUSH_PREPEND_NOT_OK( |
| ChecksumData(KsckChecksumOptions(std::move(table_filters_for_checksum_opts), |
| std::move(tablet_id_filters_for_checksum_opts))), |
| results_.error_messages, "checksum scan error"); |
| } |
| |
| // Use a special-case error if there are auth errors. This makes it harder |
| // for admins to miss that ksck isn't working right because they forgot to |
| // (e.g.) sudo -u kudu when running ksck! |
| if (std::any_of(std::begin(results_.error_messages), |
| std::end(results_.error_messages), |
| [](const Status& s) { return s.IsNotAuthorized(); })) { |
| return Status::NotAuthorized("re-run ksck with administrator privileges"); |
| } |
| if (!results_.error_messages.empty()) { |
| return Status::RuntimeError("ksck discovered errors"); |
| } |
| return Status::OK(); |
| } |
| |
| Status Ksck::CheckTabletServerUnusualFlags() { |
| int bad_servers = 0; |
| for (const auto& uuid_and_ts : cluster_->tablet_servers()) { |
| const auto& tserver = uuid_and_ts.second; |
| const auto& unusual_flags = tserver->flags(FlagsCategory::UNUSUAL); |
| if (!unusual_flags) { |
| ++bad_servers; |
| continue; |
| } |
| AddFlagsToFlagMaps(*unusual_flags, |
| tserver->address(), |
| &results_.tserver_unusual_flag_to_servers_map, |
| &results_.tserver_unusual_flag_tags_map); |
| } |
| |
| if (!results_.tserver_unusual_flag_to_servers_map.empty()) { |
| results_.warning_messages.push_back(Status::ConfigurationError( |
| "Some tablet servers have unsafe, experimental, or hidden flags set")); |
| } |
| |
| if (bad_servers > 0) { |
| return Status::Incomplete(Substitute( |
| "$0 of $1 tservers were not available to retrieve unusual flags", |
| bad_servers, cluster_->tablet_servers().size())); |
| } |
| return Status::OK(); |
| } |
| |
| Status Ksck::CheckTabletServerDivergedFlags() { |
| vector<FlagsCategory> flags_categories; |
| RETURN_NOT_OK(StringToFlagsCategories(FLAGS_flags_categories_to_check, |
| &flags_categories)); |
| for (const auto cat : flags_categories) { |
| KsckFlagToServersMap servers_by_flag; |
| size_t bad_servers = 0; |
| for (const auto& uuid_and_ts : cluster_->tablet_servers()) { |
| const auto& tserver = uuid_and_ts.second; |
| const auto& flags = tserver->flags(cat); |
| if (!flags) { |
| ++bad_servers; |
| continue; |
| } |
| AddFlagsToFlagMaps(*flags, tserver->address(), &servers_by_flag); |
| AddFlagsToFlagMaps(*flags, |
| tserver->address(), |
| &results_.tserver_checked_flag_to_servers_map); |
| } |
| for (const auto& e : servers_by_flag) { |
| if (e.second.size() + bad_servers == cluster_->tablet_servers().size()) { |
| continue; |
| } |
| results_.warning_messages.push_back(Status::ConfigurationError( |
| Substitute("Different tablet servers have different settings " |
| "for same flags of checked category '$0'", |
| FlagsCategoryToString(cat)))); |
| break; |
| } |
| if (bad_servers > 0) { |
| return Status::Incomplete(Substitute( |
| "$0 of $1 tservers were not available to retrieve $2 category flags", |
| bad_servers, |
| cluster_->tablet_servers().size(), |
| FlagsCategoryToString(cat))); |
| } |
| } |
| return Status::OK(); |
| } |
| |
| Status Ksck::CheckDivergedFlags() { |
| set<KsckFlag> masters_flags; |
| for (const auto& elem : results_.master_checked_flag_to_servers_map) { |
| InsertOrDieNoPrint(&masters_flags, elem.first); |
| } |
| set<KsckFlag> tservers_flags; |
| for (const auto& elem : results_.tserver_checked_flag_to_servers_map) { |
| InsertOrDieNoPrint(&tservers_flags, elem.first); |
| } |
| |
| vector<KsckFlag> symm_diff; |
| std::set_symmetric_difference(masters_flags.begin(), |
| masters_flags.end(), |
| tservers_flags.begin(), |
| tservers_flags.end(), |
| back_inserter(symm_diff)); |
| if (!symm_diff.empty()) { |
| for (const auto& f : symm_diff) { |
| { |
| const auto* e = FindOrNull(results_.master_checked_flag_to_servers_map, f); |
| if (e) { |
| InsertOrDieNoPrint(&results_.master_diverged_flag_to_servers_map, f, *e); |
| continue; |
| } |
| } |
| { |
| const auto* e = FindOrNull(results_.tserver_checked_flag_to_servers_map, f); |
| if (e) { |
| InsertOrDieNoPrint(&results_.tserver_diverged_flag_to_servers_map, f, *e); |
| continue; |
| } |
| } |
| // The flag/value pair must be either of masters' or tservers'. |
| LOG(DFATAL) << "found neither masters' or tservers' flag: " << f.first; |
| } |
| |
| results_.warning_messages.push_back(Status::ConfigurationError( |
| "Same flags have different values between masters and tablet servers " |
| "for at least one checked flag category")); |
| } |
| return Status::OK(); |
| } |
| |
| Status Ksck::CheckServerVersions() { |
| results_.version_summaries.clear(); |
| for (const auto& s : results_.cluster_status.master_summaries) { |
| if (!s.version) continue; |
| const auto& server = Substitute("master@$0", s.address); |
| auto& servers = LookupOrInsert(&results_.version_summaries, *s.version, {}); |
| servers.push_back(server); |
| } |
| for (const auto& s : results_.cluster_status.tserver_summaries) { |
| if (!s.version) continue; |
| const auto& server = Substitute("tserver@$0", s.address); |
| auto& servers = LookupOrInsert(&results_.version_summaries, *s.version, {}); |
| servers.push_back(server); |
| } |
| if (results_.version_summaries.size() > 1) { |
| // This status seemed to fit best even though a version mismatch isn't an |
| // error. In any case, ksck only prints the message for warnings. |
| return Status::ConfigurationError( |
| Substitute("not all servers are running the same version: " |
| "$0 different versions were seen", |
| results_.version_summaries.size())); |
| } |
| return Status::OK(); |
| } |
| |
| Status Ksck::PrintResults() { |
| PrintMode mode; |
| if (FLAGS_ksck_format == "plain_concise") { |
| mode = PrintMode::PLAIN_CONCISE; |
| } else if (FLAGS_ksck_format == "plain_full") { |
| mode = PrintMode::PLAIN_FULL; |
| } else if (FLAGS_ksck_format == "json_pretty") { |
| mode = PrintMode::JSON_PRETTY; |
| } else if (FLAGS_ksck_format == "json_compact") { |
| mode = PrintMode::JSON_COMPACT; |
| } else { |
| return Status::InvalidArgument("unknown ksck format (--ksck_format)", |
| FLAGS_ksck_format); |
| } |
| return results_.PrintTo(mode, print_sections_flags_, *out_); |
| } |
| |
| Status Ksck::RunAndPrintResults() { |
| Status s = Run(); |
| RETURN_NOT_OK_PREPEND(PrintResults(), "error printing results"); |
| return s; |
| } |
| |
| Status Ksck::CheckTablesConsistency() { |
| int bad_tables_count = 0; |
| auto& cluster_status = results_.cluster_status; |
| if (cluster_->txn_sys_table()) { |
| if (!VerifyTable(cluster_->txn_sys_table(), &cluster_status.system_table_summaries)) { |
| bad_tables_count++; |
| } |
| } |
| for (const shared_ptr<KsckTable> &table : cluster_->tables()) { |
| if (!VerifyTable(table, &cluster_status.table_summaries)) { |
| bad_tables_count++; |
| } |
| } |
| |
| if (bad_tables_count > 0) { |
| return Status::Corruption( |
| Substitute("$0 out of $1 table(s) are not healthy", bad_tables_count, |
| cluster_status.table_summaries.size() + |
| cluster_status.system_table_summaries.size())); |
| } |
| return Status::OK(); |
| } |
| |
| Status Ksck::ChecksumData(const KsckChecksumOptions& opts) { |
| KsckChecksummer checksummer(cluster_.get()); |
| auto* checksum_results = &results_.checksum_results; |
| // Don't ruin JSON output with ad hoc progress updates. |
| auto* out_for_progress_updates = IsNonJSONFormat() ? out_ : nullptr; |
| return checksummer.ChecksumData(opts, |
| checksum_results, |
| out_for_progress_updates); |
| } |
| |
| bool Ksck::VerifyTable(const shared_ptr<KsckTable>& table, vector<TableSummary>* table_summaries) { |
| if (table->tablets().empty()) { |
| VLOG(1) << Substitute("Skipping table $0 as it has no matching tablets", |
| table->name()); |
| return true; |
| } |
| |
| TableSummary ts; |
| ts.id = table->id(); |
| ts.name = table->name(); |
| ts.replication_factor = table->num_replicas(); |
| VLOG(1) << Substitute("Verifying $0 tablet(s) for table $1 configured with num_replicas = $2", |
| table->tablets().size(), table->name(), table->num_replicas()); |
| for (const auto& tablet : table->tablets()) { |
| auto tablet_result = VerifyTablet(tablet, table->num_replicas()); |
| switch (tablet_result) { |
| case HealthCheckResult::HEALTHY: |
| ts.healthy_tablets++; |
| break; |
| case HealthCheckResult::RECOVERING: |
| ts.recovering_tablets++; |
| break; |
| case HealthCheckResult::UNDER_REPLICATED: |
| ts.underreplicated_tablets++; |
| break; |
| case HealthCheckResult::CONSENSUS_MISMATCH: |
| ts.consensus_mismatch_tablets++; |
| break; |
| case HealthCheckResult::UNAVAILABLE: |
| ts.unavailable_tablets++; |
| break; |
| } |
| } |
| bool all_healthy = ts.healthy_tablets == ts.TotalTablets(); |
| if (ts.TotalTablets() > 0) { |
| table_summaries->emplace_back(std::move(ts)); |
| } |
| return all_healthy; |
| } |
| |
| HealthCheckResult Ksck::VerifyTablet(const shared_ptr<KsckTablet>& tablet, |
| int table_num_replicas) { |
| const string tablet_str = Substitute("Tablet $0 of table '$1'", |
| tablet->id(), tablet->table()->name()); |
| |
| auto leader_it = std::find_if(tablet->replicas().cbegin(), tablet->replicas().cend(), |
| [](const shared_ptr<KsckTabletReplica>& r) { return r->is_leader(); }); |
| optional<string> leader_uuid; |
| if (leader_it != tablet->replicas().cend()) { |
| leader_uuid = (*leader_it)->ts_uuid(); |
| } |
| vector<string> voter_uuids_from_master; |
| vector<string> non_voter_uuids_from_master; |
| for (const auto& replica : tablet->replicas()) { |
| if (replica->is_voter()) { |
| voter_uuids_from_master.push_back(replica->ts_uuid()); |
| } else { |
| non_voter_uuids_from_master.push_back(replica->ts_uuid()); |
| } |
| } |
| ConsensusState master_config(ConsensusConfigType::MASTER, |
| nullopt, |
| nullopt, |
| leader_uuid, |
| voter_uuids_from_master, |
| non_voter_uuids_from_master); |
| |
| int leaders_count = 0; |
| int running_voters_count = 0; |
| int copying_replicas_count = 0; |
| int conflicting_states = 0; |
| int num_voters = 0; |
| vector<ReplicaSummary> replicas; |
| for (const shared_ptr<KsckTabletReplica>& replica : tablet->replicas()) { |
| replicas.emplace_back(); |
| auto* repl_info = &replicas.back(); |
| repl_info->ts_uuid = replica->ts_uuid(); |
| VLOG(1) << Substitute("A replica of tablet $0 is on live tablet server $1", |
| tablet->id(), replica->ts_uuid()); |
| |
| // Check for agreement on tablet assignment and state between the master |
| // and the tablet server. |
| auto ts = FindPointeeOrNull(cluster_->tablet_servers(), replica->ts_uuid()); |
| if (ts) { |
| repl_info->ts_address = ts->address(); |
| } |
| if (ts && ts->is_healthy()) { |
| repl_info->ts_healthy = true; |
| repl_info->state = ts->ReplicaState(tablet->id()); |
| if (ContainsKey(ts->tablet_status_map(), tablet->id())) { |
| repl_info->status_pb = ts->tablet_status_map().at(tablet->id()); |
| } |
| |
| // Organize consensus info for each replica. |
| pair<string, string> tablet_key = std::make_pair(ts->uuid(), tablet->id()); |
| if (ContainsKey(ts->tablet_consensus_state_map(), tablet_key)) { |
| const auto& cstate = FindOrDieNoPrint(ts->tablet_consensus_state_map(), tablet_key); |
| ConsensusState ksck_cstate; |
| BuildConsensusStateForConfigMember(cstate, &ksck_cstate); |
| repl_info->consensus_state = std::move(ksck_cstate); |
| } |
| } |
| |
| repl_info->is_leader = replica->is_leader(); |
| repl_info->is_voter = replica->is_voter(); |
| num_voters += replica->is_voter() ? 1 : 0; |
| if (replica->is_leader()) { |
| leaders_count++; |
| } |
| if (repl_info->state == tablet::RUNNING && replica->is_voter()) { |
| running_voters_count++; |
| } else if (repl_info->status_pb && |
| repl_info->status_pb->tablet_data_state() == tablet::TABLET_DATA_COPYING) { |
| copying_replicas_count++; |
| } |
| } |
| // Compare the master's and peers' consensus configs. |
| for (const auto& r : replicas) { |
| if (r.consensus_state && !r.consensus_state->Matches(master_config)) { |
| conflicting_states++; |
| } |
| } |
| |
| // Determine the overall health state of the tablet. |
| HealthCheckResult result = HealthCheckResult::HEALTHY; |
| string status; |
| int majority_size = consensus::MajoritySize(num_voters); |
| if (copying_replicas_count > 0) { |
| result = HealthCheckResult::RECOVERING; |
| status = Substitute("$0 is $1: $2 on-going tablet copies", |
| tablet_str, |
| Color(AnsiCode::YELLOW, "recovering"), |
| copying_replicas_count); |
| } else if (running_voters_count < majority_size) { |
| result = HealthCheckResult::UNAVAILABLE; |
| status = Substitute("$0 is $1: $2 replica(s) not RUNNING", |
| tablet_str, |
| Color(AnsiCode::RED, "unavailable"), |
| num_voters - running_voters_count); |
| } else if (running_voters_count < num_voters) { |
| result = HealthCheckResult::UNDER_REPLICATED; |
| status = Substitute("$0 is $1: $2 replica(s) not RUNNING", |
| tablet_str, |
| Color(AnsiCode::YELLOW, "under-replicated"), |
| num_voters - running_voters_count); |
| } else if (check_replica_count_ && num_voters < table_num_replicas) { |
| result = HealthCheckResult::UNDER_REPLICATED; |
| status = Substitute("$0 is $1: configuration has $2 replicas vs desired $3", |
| tablet_str, |
| Color(AnsiCode::YELLOW, "under-replicated"), |
| num_voters, |
| table_num_replicas); |
| } else if (leaders_count != 1) { |
| result = HealthCheckResult::UNAVAILABLE; |
| status = Substitute("$0 is $1: expected one LEADER replica", |
| tablet_str, Color(AnsiCode::RED, "unavailable")); |
| } else if (conflicting_states > 0) { |
| result = HealthCheckResult::CONSENSUS_MISMATCH; |
| status = Substitute("$0 is $1: $2 replicas' active configs disagree with the " |
| "leader master's", |
| tablet_str, |
| Color(AnsiCode::YELLOW, "conflicted"), |
| conflicting_states); |
| } else { |
| status = Substitute("$0 is $1.", |
| tablet_str, |
| Color(AnsiCode::GREEN, "healthy")); |
| } |
| |
| TabletSummary tablet_summary; |
| tablet_summary.id = tablet->id(); |
| tablet_summary.table_id = tablet->table()->id(); |
| tablet_summary.table_name = tablet->table()->name(); |
| tablet_summary.result = result; |
| tablet_summary.status = status; |
| tablet_summary.master_cstate = std::move(master_config); |
| tablet_summary.replicas.swap(replicas); |
| |
| // Add printable representation of the key for the start of the range. |
| const auto& range_key_begin = tablet->partition().begin().range_key(); |
| ostringstream ss_range_key_begin; |
| for (size_t i = 0; i < range_key_begin.size(); ++i) { |
| ss_range_key_begin << std::hex << std::setw(2) << std::setfill('0') |
| << static_cast<uint16_t>(range_key_begin[i]); |
| } |
| tablet_summary.range_key_begin = ss_range_key_begin.str(); |
| VLOG(1) << Substitute("range start key for tablet $0: '$1'", |
| tablet_summary.id, tablet_summary.range_key_begin); |
| results_.cluster_status.tablet_summaries.push_back(std::move(tablet_summary)); |
| return result; |
| } |
| |
| } // namespace tools |
| } // namespace kudu |