| // Licensed to the Apache Software Foundation (ASF) under one |
| // or more contributor license agreements. See the NOTICE file |
| // distributed with this work for additional information |
| // regarding copyright ownership. The ASF licenses this file |
| // to you under the Apache License, Version 2.0 (the |
| // "License"); you may not use this file except in compliance |
| // with the License. You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, |
| // software distributed under the License is distributed on an |
| // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| // KIND, either express or implied. See the License for the |
| // specific language governing permissions and limitations |
| // under the License. |
| |
| #include <fstream> // IWYU pragma: keep |
| #include <functional> |
| #include <iostream> |
| #include <memory> |
| #include <optional> |
| #include <string> |
| #include <unordered_map> |
| #include <utility> |
| #include <vector> |
| |
| #include <gflags/gflags.h> |
| |
| #include "kudu/client/client.h" |
| #include "kudu/client/shared_ptr.h" // IWYU pragma: keep |
| #include "kudu/common/wire_protocol.h" |
| #include "kudu/consensus/consensus.pb.h" |
| #include "kudu/consensus/metadata.pb.h" |
| #include "kudu/gutil/map-util.h" |
| #include "kudu/gutil/strings/split.h" |
| #include "kudu/gutil/strings/substitute.h" |
| #include "kudu/master/master.pb.h" |
| #include "kudu/master/master.proxy.h" |
| #include "kudu/rpc/response_callback.h" |
| #include "kudu/tools/tool_action.h" |
| #include "kudu/tools/tool_action_common.h" |
| #include "kudu/tools/tool_replica_util.h" |
| #include "kudu/util/monotime.h" |
| #include "kudu/util/net/net_util.h" |
| #include "kudu/util/status.h" |
| #include "kudu/util/string_case.h" |
| |
| DEFINE_bool(abrupt, false, |
| "Whether the leader should step down without attempting to " |
| "transfer leadership gracefully. A graceful transfer minimizes " |
| "delays in tablet operations, but will fail if the tablet cannot " |
| "arrange a successor."); |
| DEFINE_string(new_leader_uuid, "", |
| "UUID of the server that leadership should be transferred to. " |
| "Leadership may only be transferred to a voting member of the " |
| "leader's active config. If the designated successor cannot " |
| "catch up to the leader within one election timeout, leadership " |
| "transfer will not occur. If blank, the leader chooses its own " |
| "successor, attempting to transfer leadership as soon as " |
| "possible. This cannot be set if --abrupt is set."); |
| DEFINE_int64(move_copy_timeout_sec, 600, |
| "Number of seconds to wait for tablet copy to complete when relocating a tablet"); |
| DEFINE_int64(move_leader_timeout_sec, 30, |
| "Number of seconds to wait for a leader when relocating a leader tablet"); |
| |
| using kudu::client::KuduClient; |
| using kudu::consensus::ADD_PEER; |
| using kudu::consensus::ChangeConfigType; |
| using kudu::consensus::LeaderStepDownMode; |
| using kudu::consensus::MODIFY_PEER; |
| using kudu::consensus::RaftPeerPB; |
| using kudu::master::MasterServiceProxy; |
| using kudu::master::ReplaceTabletRequestPB; |
| using kudu::master::ReplaceTabletResponsePB; |
| using std::cerr; |
| using std::cout; |
| using std::endl; |
| using std::make_optional; |
| using std::nullopt; |
| using std::optional; |
| using std::string; |
| using std::unique_ptr; |
| using std::vector; |
| using strings::Split; |
| using strings::Substitute; |
| |
| namespace kudu { |
| namespace tools { |
| |
| namespace { |
| |
| const char* const kReplicaTypeArg = "replica_type"; |
| const char* const kTsUuidArg = "ts_uuid"; |
| const char* const kFromTsUuidArg = "from_ts_uuid"; |
| const char* const kToTsUuidArg = "to_ts_uuid"; |
| |
| Status WaitForCleanKsck(const vector<string>& master_addresses, |
| const string& tablet_id, |
| const MonoDelta& timeout) { |
| Status s; |
| MonoTime deadline = MonoTime::Now() + timeout; |
| while (MonoTime::Now() < deadline) { |
| s = DoKsckForTablet(master_addresses, tablet_id); |
| if (s.ok()) return s; |
| SleepFor(MonoDelta::FromMilliseconds(1000)); |
| } |
| return s.CloneAndPrepend("timed out with ksck errors remaining: last error"); |
| } |
| |
| Status ChangeConfig(const RunnerContext& context, ChangeConfigType cc_type) { |
| const string& tablet_id = FindOrDie(context.required_args, kTabletIdArg); |
| const string& replica_uuid = FindOrDie(context.required_args, kTsUuidArg); |
| optional<RaftPeerPB::MemberType> member_type; |
| if (cc_type == consensus::ADD_PEER || cc_type == consensus::MODIFY_PEER) { |
| const string& replica_type = FindOrDie(context.required_args, kReplicaTypeArg); |
| string uppercase_peer_type; |
| ToUpperCase(replica_type, &uppercase_peer_type); |
| RaftPeerPB::MemberType member_type_val; |
| if (!RaftPeerPB::MemberType_Parse(uppercase_peer_type, &member_type_val)) { |
| return Status::InvalidArgument("Unrecognized peer type", replica_type); |
| } |
| member_type = member_type_val; |
| } |
| |
| vector<string> master_addresses; |
| RETURN_NOT_OK(ParseMasterAddresses(context, &master_addresses)); |
| return DoChangeConfig(master_addresses, tablet_id, replica_uuid, member_type, cc_type); |
| } |
| |
| Status AddReplica(const RunnerContext& context) { |
| return ChangeConfig(context, consensus::ADD_PEER); |
| } |
| |
| Status ChangeReplicaType(const RunnerContext& context) { |
| return ChangeConfig(context, consensus::MODIFY_PEER); |
| } |
| |
| Status RemoveReplica(const RunnerContext& context) { |
| return ChangeConfig(context, consensus::REMOVE_PEER); |
| } |
| |
| Status LeaderStepDown(const RunnerContext& context) { |
| vector<string> master_addresses; |
| RETURN_NOT_OK(ParseMasterAddresses(context, &master_addresses)); |
| |
| const string& tablet_id = FindOrDie(context.required_args, kTabletIdArg); |
| const LeaderStepDownMode mode = FLAGS_abrupt ? LeaderStepDownMode::ABRUPT : |
| LeaderStepDownMode::GRACEFUL; |
| const optional<string> new_leader_uuid = |
| FLAGS_new_leader_uuid.empty() ? nullopt |
| : make_optional(FLAGS_new_leader_uuid); |
| if (mode == LeaderStepDownMode::ABRUPT && new_leader_uuid) { |
| return Status::InvalidArgument("cannot specify both --new_leader_uuid and --abrupt"); |
| } |
| |
| client::sp::shared_ptr<KuduClient> client; |
| RETURN_NOT_OK(CreateKuduClient(master_addresses, &client)); |
| |
| string leader_uuid; |
| HostPort leader_hp; |
| bool no_leader = false; |
| Status s = GetTabletLeader(client, tablet_id, |
| &leader_uuid, &leader_hp, &no_leader); |
| if (s.IsNotFound() && no_leader) { |
| // If leadership should be transferred to a specific node, exit with an |
| // error if there's no leader since we can't orchestrate the transfer. |
| if (new_leader_uuid) { |
| return s.CloneAndPrepend( |
| Substitute("unable to transfer leadership to $0", *new_leader_uuid)); |
| } |
| // Otherwise, a new election should happen soon, which will achieve |
| // something like what the client wanted, so we'll exit gracefully. |
| cout << s.ToString() << endl; |
| return Status::OK(); |
| } |
| RETURN_NOT_OK(s); |
| |
| // If the requested new leader is the leader, the command can short-circuit. |
| if (new_leader_uuid && (leader_uuid == *new_leader_uuid)) { |
| cout << Substitute("Requested new leader $0 is already the leader", |
| leader_uuid) << endl; |
| return Status::OK(); |
| } |
| |
| return DoLeaderStepDown(tablet_id, leader_uuid, leader_hp, |
| mode, new_leader_uuid, |
| client->default_admin_operation_timeout()); |
| } |
| |
| Status WaitForMoveToComplete(const vector<string>& master_addresses, |
| client::sp::shared_ptr<KuduClient> client, |
| const string& tablet_id, |
| const string& from_ts_uuid, |
| const string& to_ts_uuid, |
| MonoDelta copy_timeout) { |
| // Wait until the tablet copy completes and the tablet returns to perfect health. |
| MonoTime start = MonoTime::Now(); |
| MonoTime deadline = start + copy_timeout; |
| while (MonoTime::Now() < deadline) { |
| bool is_completed = false; |
| Status move_completion_status; |
| RETURN_NOT_OK(CheckCompleteMove(master_addresses, client, tablet_id, |
| from_ts_uuid, to_ts_uuid, |
| &is_completed, &move_completion_status)); |
| if (is_completed) { |
| return move_completion_status; |
| } |
| SleepFor(MonoDelta::FromMilliseconds(500)); |
| } |
| return Status::TimedOut(Substitute("unable to complete tablet replica move after $0", |
| (MonoTime::Now() - start).ToString())); |
| } |
| |
| Status MoveReplica(const RunnerContext& context) { |
| vector<string> master_addresses; |
| RETURN_NOT_OK(ParseMasterAddresses(context, &master_addresses)); |
| const string& tablet_id = FindOrDie(context.required_args, kTabletIdArg); |
| const string& from_ts_uuid = FindOrDie(context.required_args, kFromTsUuidArg); |
| const string& to_ts_uuid = FindOrDie(context.required_args, kToTsUuidArg); |
| |
| // Check the tablet is in perfect health first. |
| // We retry since occasionally ksck returns bad status because of transient |
| // issues like leader elections. |
| RETURN_NOT_OK_PREPEND(WaitForCleanKsck(master_addresses, |
| tablet_id, |
| MonoDelta::FromSeconds(5)), |
| "ksck pre-move health check failed"); |
| client::sp::shared_ptr<KuduClient> client; |
| RETURN_NOT_OK(CreateKuduClient(master_addresses, &client)); |
| RETURN_NOT_OK(ScheduleReplicaMove(master_addresses, client, |
| tablet_id, from_ts_uuid, to_ts_uuid)); |
| const auto copy_timeout = MonoDelta::FromSeconds(FLAGS_move_copy_timeout_sec); |
| return WaitForMoveToComplete(master_addresses, client, tablet_id, |
| from_ts_uuid, to_ts_uuid, copy_timeout); |
| |
| } |
| |
| Status ReplaceTablet(const RunnerContext& context) { |
| const string& tablet_id = FindOrDie(context.required_args, kTabletIdArg); |
| |
| LeaderMasterProxy proxy; |
| RETURN_NOT_OK(proxy.Init(context)); |
| |
| ReplaceTabletRequestPB req; |
| ReplaceTabletResponsePB resp; |
| req.set_tablet_id(tablet_id); |
| Status s = proxy.SyncRpc<ReplaceTabletRequestPB, ReplaceTabletResponsePB>( |
| req, &resp, "ReplaceTablet", &MasterServiceProxy::ReplaceTabletAsync); |
| RETURN_NOT_OK(s); |
| |
| if (resp.has_error()) { |
| s = StatusFromPB(resp.error().status()); |
| return s.CloneAndPrepend(Substitute("unable to replace tablet $0", tablet_id)); |
| } |
| cerr << "Replaced tablet " << tablet_id << " with tablet "; |
| cout << resp.replacement_tablet_id() << endl; |
| return Status::OK(); |
| } |
| |
| } // anonymous namespace |
| |
| unique_ptr<Mode> BuildTabletMode() { |
| unique_ptr<Action> add_replica = |
| ClusterActionBuilder("add_replica", &AddReplica) |
| .Description("Add a new replica to a tablet's Raft configuration") |
| .AddRequiredParameter({ kTabletIdArg, kTabletIdArgDesc }) |
| .AddRequiredParameter({ kTsUuidArg, |
| "UUID of the tablet server that should host the new replica" }) |
| .AddRequiredParameter( |
| { kReplicaTypeArg, "New replica's type. Must be VOTER or NON_VOTER." |
| }) |
| .Build(); |
| |
| unique_ptr<Action> change_replica_type = |
| ClusterActionBuilder("change_replica_type", &ChangeReplicaType) |
| .Description( |
| "Change the type of an existing replica in a tablet's Raft configuration") |
| .AddRequiredParameter({ kTabletIdArg, kTabletIdArgDesc }) |
| .AddRequiredParameter({ kTsUuidArg, |
| "UUID of the tablet server hosting the existing replica" }) |
| .AddRequiredParameter( |
| { kReplicaTypeArg, "Existing replica's new type. Must be VOTER or NON_VOTER." |
| }) |
| .Build(); |
| |
| unique_ptr<Action> remove_replica = |
| ClusterActionBuilder("remove_replica", &RemoveReplica) |
| .Description("Remove an existing replica from a tablet's Raft configuration") |
| .AddRequiredParameter({ kTabletIdArg, kTabletIdArgDesc }) |
| .AddRequiredParameter({ kTsUuidArg, |
| "UUID of the tablet server hosting the existing replica" }) |
| .Build(); |
| |
| const string move_extra_desc = "The replica move tool effectively moves a " |
| "replica from one tablet server to another by adding a replica to the " |
| "new server and then removing it from the old one. It requires that " |
| "ksck return no errors when run against the target tablet. If the move " |
| "fails, the user should wait for any tablet copy to complete, and, if " |
| "the copy succeeds, use remove_replica manually. If the copy fails, the " |
| "new replica will be deleted automatically after some time, and then the " |
| "move can be retried."; |
| unique_ptr<Action> move_replica = |
| ClusterActionBuilder("move_replica", &MoveReplica) |
| .Description("Move a tablet replica from one tablet server to another") |
| .ExtraDescription(move_extra_desc) |
| .AddRequiredParameter({ kTabletIdArg, kTabletIdArgDesc }) |
| .AddRequiredParameter({ kFromTsUuidArg, "UUID of the tablet server to move from" }) |
| .AddRequiredParameter({ kToTsUuidArg, "UUID of the tablet server to move to" }) |
| .Build(); |
| |
| unique_ptr<Action> leader_step_down = |
| ClusterActionBuilder("leader_step_down", &LeaderStepDown) |
| .Description("Change the tablet's leader") |
| .AddRequiredParameter({ kTabletIdArg, kTabletIdArgDesc }) |
| .AddOptionalParameter("abrupt") |
| .AddOptionalParameter("new_leader_uuid") |
| .Build(); |
| |
| unique_ptr<Mode> change_config = |
| ModeBuilder("change_config") |
| .Description("Change a tablet's Raft configuration") |
| .AddAction(std::move(add_replica)) |
| .AddAction(std::move(change_replica_type)) |
| .AddAction(std::move(move_replica)) |
| .AddAction(std::move(remove_replica)) |
| .Build(); |
| |
| unique_ptr<Action> replace_tablet = |
| ClusterActionBuilder("unsafe_replace_tablet", &ReplaceTablet) |
| .Description("Replace a tablet with an empty one, deleting the previous tablet.") |
| .ExtraDescription("Use this tool to repair a table when one of its tablets has permanently " |
| "lost all of its replicas. It replaces the unrecoverable tablet with a new " |
| "empty one representing the same partition. Its primary use is to jettison " |
| "an unrecoverable tablet in order to make the rest of the table " |
| "available.\n\n" |
| "NOTE: The original tablet will be deleted. Its data will be permanently " |
| "lost. Additionally, clients should be restarted before attempting to " |
| "use the repaired table (see KUDU-2376).") |
| .AddRequiredParameter({ kTabletIdArg, kTabletIdArgDesc }) |
| .Build(); |
| |
| return ModeBuilder("tablet") |
| .Description("Operate on remote Kudu tablets") |
| .AddMode(std::move(change_config)) |
| .AddAction(std::move(leader_step_down)) |
| .AddAction(std::move(replace_tablet)) |
| .Build(); |
| } |
| |
| } // namespace tools |
| } // namespace kudu |
| |