src/kudu/consensus/quorum_util-test.cc

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

#include "kudu/consensus/quorum_util.h"

#include <initializer_list>
#include <memory>
#include <string>
#include <utility>
#include <vector>

#include <boost/optional/optional.hpp>
#include <gtest/gtest.h>

#include "kudu/common/common.pb.h"
#include "kudu/consensus/metadata.pb.h"
#include "kudu/gutil/strings/substitute.h"
#include "kudu/util/status.h"
#include "kudu/util/test_macros.h"

using std::pair;
using std::string;
using std::unique_ptr;
using std::vector;
using strings::Substitute;

namespace kudu {
namespace consensus {

// Handy notation of membership types used by AddPeer(), etc.
constexpr auto N = RaftPeerPB::NON_VOTER;           // NOLINT(readability-identifier-naming)
constexpr auto U = RaftPeerPB::UNKNOWN_MEMBER_TYPE; // NOLINT(readability-identifier-naming)
constexpr auto V = RaftPeerPB::VOTER;               // NOLINT(readability-identifier-naming)

// The various possible health statuses.
const auto kHealthStatuses = { '?', '-', 'x', '+' };

typedef std::pair<string, bool> Attr;

static void SetOverallHealth(HealthReportPB* health_report,
                             char overall_health) {
  switch (overall_health) {
    case '+':
      health_report->set_overall_health(HealthReportPB::HEALTHY);
      break;
    case '-':
      health_report->set_overall_health(HealthReportPB::FAILED);
      break;
    case 'x':
      health_report->set_overall_health(HealthReportPB::FAILED_UNRECOVERABLE);
      break;
    case '?':
      health_report->set_overall_health(HealthReportPB::UNKNOWN);
      break;
    default:
      FAIL() << overall_health << ": unexpected replica health status";
      break;
  }
}

// Add a consensus peer into the specified configuration.
static void AddPeer(RaftConfigPB* config,
                    const string& uuid,
                    RaftPeerPB::MemberType type,
                    boost::optional<char> overall_health = boost::none,
                    vector<Attr> attrs = {}) {
  RaftPeerPB* peer = config->add_peers();
  peer->set_permanent_uuid(uuid);
  peer->mutable_last_known_addr()->set_host(uuid + ".example.com");
  peer->set_member_type(type);
  if (overall_health) {
    unique_ptr<HealthReportPB> health_report(new HealthReportPB);
    SetOverallHealth(health_report.get(), *overall_health);
    peer->set_allocated_health_report(health_report.release());
  }
  if (!attrs.empty()) {
    unique_ptr<RaftPeerAttrsPB> attrs_pb(new RaftPeerAttrsPB);
    for (const auto& attr : attrs) {
      if (attr.first == "PROMOTE") {
        attrs_pb->set_promote(attr.second);
      } else if (attr.first == "REPLACE") {
        attrs_pb->set_replace(attr.second);
      } else {
        FAIL() << attr.first << ": unexpected attribute to set";
      }
    }
    peer->set_allocated_attrs(attrs_pb.release());
  }
}

using RaftMemberSpec = pair<string, RaftPeerPB::MemberType>;

static RaftConfigPB CreateConfig(const vector<RaftMemberSpec>& specs) {
  RaftConfigPB config;
  for (const auto& spec : specs) {
    AddPeer(&config, spec.first, spec.second);
  }
  return config;
}

static void PromotePeer(RaftConfigPB* config, const string& peer_uuid) {
  RaftPeerPB* peer_pb;
  const Status s = GetRaftConfigMember(config, peer_uuid, &peer_pb);
  if (!s.ok()) {
    FAIL() << peer_uuid << ": " << s.ToString();
  }
  peer_pb->set_member_type(V);
  //peer_pb->mutable_attrs()->clear_promote();
  peer_pb->mutable_attrs()->set_promote(false);
}

static void RemovePeer(RaftConfigPB* config, const string& peer_uuid) {
  if (!RemoveFromRaftConfig(config, peer_uuid)) {
    FAIL() << peer_uuid << ": peer is not in the config";
  }
}

static void SetPeerHealth(RaftConfigPB* config, const string& uuid, char health) {
  RaftPeerPB* peer_pb;
  const Status s = GetRaftConfigMember(config, uuid, &peer_pb);
  if (!s.ok()) {
    FAIL() << "unexpected failure from GetRaftConfigMember(): " << s.ToString();
  }
  SetOverallHealth(peer_pb->mutable_health_report(), health);
}

TEST(QuorumUtilTest, TestMemberExtraction) {
  RaftConfigPB config;
  AddPeer(&config, "A", V);
  AddPeer(&config, "B", V);
  AddPeer(&config, "C", V);

  // Basic test for GetRaftConfigMember().
  RaftPeerPB* peer_pb;
  Status s = GetRaftConfigMember(&config, "invalid", &peer_pb);
  ASSERT_TRUE(s.IsNotFound()) << s.ToString();
  ASSERT_OK(GetRaftConfigMember(&config, "A", &peer_pb));
  ASSERT_EQ("A", peer_pb->permanent_uuid());

  // Basic test for GetRaftConfigLeader().
  ConsensusStatePB cstate;
  *cstate.mutable_committed_config() = config;
  s = GetRaftConfigLeader(&cstate, &peer_pb);
  ASSERT_TRUE(s.IsNotFound()) << s.ToString();
  cstate.set_leader_uuid("B");
  ASSERT_OK(GetRaftConfigLeader(&cstate, &peer_pb));
  ASSERT_EQ("B", peer_pb->permanent_uuid());
}

TEST(QuorumUtilTest, TestDiffConsensusStates) {
  ConsensusStatePB old_cs;
  AddPeer(old_cs.mutable_committed_config(), "A", V);
  AddPeer(old_cs.mutable_committed_config(), "B", V);
  AddPeer(old_cs.mutable_committed_config(), "C", V);
  old_cs.set_current_term(1);
  old_cs.set_leader_uuid("A");
  old_cs.mutable_committed_config()->set_opid_index(1);

  // Simple case of no change.
  EXPECT_EQ("no change",
            DiffConsensusStates(old_cs, old_cs));

  // Simulate a leader change.
  {
    auto new_cs = old_cs;
    new_cs.set_leader_uuid("B");
    new_cs.set_current_term(2);

    EXPECT_EQ("term changed from 1 to 2, "
              "leader changed from A (A.example.com) to B (B.example.com)",
              DiffConsensusStates(old_cs, new_cs));
  }

  // Simulate eviction of a peer.
  {
    auto new_cs = old_cs;
    new_cs.mutable_committed_config()->set_opid_index(2);
    new_cs.mutable_committed_config()->mutable_peers()->RemoveLast();

    EXPECT_EQ("config changed from index 1 to 2, "
              "VOTER C (C.example.com) evicted",
              DiffConsensusStates(old_cs, new_cs));
  }

  // Simulate addition of a peer.
  {
    auto new_cs = old_cs;
    new_cs.mutable_committed_config()->set_opid_index(2);
    AddPeer(new_cs.mutable_committed_config(), "D", N);

    EXPECT_EQ("config changed from index 1 to 2, "
              "NON_VOTER D (D.example.com) added",
              DiffConsensusStates(old_cs, new_cs));
  }

  // Simulate change of a peer's member type.
  {
    auto new_cs = old_cs;
    new_cs.mutable_committed_config()->set_opid_index(2);
    new_cs.mutable_committed_config()
      ->mutable_peers()->Mutable(2)->set_member_type(N);

    EXPECT_EQ("config changed from index 1 to 2, "
              "C (C.example.com) changed from VOTER to NON_VOTER",
              DiffConsensusStates(old_cs, new_cs));
  }

  // Simulate change from no leader to a leader
  {
    auto no_leader_cs = old_cs;
    no_leader_cs.clear_leader_uuid();
    auto new_cs = old_cs;
    new_cs.set_current_term(2);

    EXPECT_EQ("term changed from 1 to 2, "
              "leader changed from <none> to A (A.example.com)",
              DiffConsensusStates(no_leader_cs, new_cs));
  }

  // Simulate gaining a pending config
  {
    auto pending_config_cs = old_cs;
    pending_config_cs.mutable_pending_config();
    EXPECT_EQ("now has a pending config: ", DiffConsensusStates(old_cs, pending_config_cs));
  }

  // Simulate losing a pending config
  {
    auto pending_config_cs = old_cs;
    pending_config_cs.mutable_pending_config();
    EXPECT_EQ("no longer has a pending config: ", DiffConsensusStates(pending_config_cs, old_cs));
  }

  // Simulate a change in a pending config
  {
    auto before_cs = old_cs;
    AddPeer(before_cs.mutable_pending_config(), "A", V);
    auto after_cs = before_cs;
    after_cs.mutable_pending_config()
      ->mutable_peers()->Mutable(0)->set_member_type(N);

    EXPECT_EQ("pending config changed, A (A.example.com) changed from VOTER to NON_VOTER",
              DiffConsensusStates(before_cs, after_cs));
  }
}

// Unit test for the variants of GetConsensusRole().
TEST(QuorumUtilTest, TestGetConsensusRole) {
  const auto LEADER = RaftPeerPB::LEADER;
  const auto FOLLOWER = RaftPeerPB::FOLLOWER;
  const auto LEARNER = RaftPeerPB::LEARNER;
  const auto NON_PARTICIPANT = RaftPeerPB::NON_PARTICIPANT;

  // 3-argument variant of GetConsensusRole().
  const auto config1 = CreateConfig({ {"A", V}, {"B", V}, {"C", N} });
  ASSERT_EQ(LEADER, GetConsensusRole("A", "A", config1));
  ASSERT_EQ(FOLLOWER, GetConsensusRole("B", "A", config1));
  ASSERT_EQ(FOLLOWER, GetConsensusRole("A", "", config1));
  ASSERT_EQ(LEARNER, GetConsensusRole("C", "A", config1));
  ASSERT_EQ(LEARNER, GetConsensusRole("C", "C", config1)); // Illegal.
  ASSERT_EQ(NON_PARTICIPANT, GetConsensusRole("D", "A", config1));
  ASSERT_EQ(NON_PARTICIPANT, GetConsensusRole("D", "D", config1)); // Illegal.
  ASSERT_EQ(NON_PARTICIPANT, GetConsensusRole("", "A", config1)); // Illegal.
  ASSERT_EQ(NON_PARTICIPANT, GetConsensusRole("", "", config1)); // Illegal.

  // 2-argument variant of GetConsensusRole().
  const auto config2 = CreateConfig({ {"A", V}, {"B", V}, {"C", V} });
  ConsensusStatePB cstate;
  *cstate.mutable_committed_config() = config1;
  *cstate.mutable_pending_config() = config2;
  cstate.set_leader_uuid("A");
  ASSERT_EQ(LEADER, GetConsensusRole("A", cstate));
  ASSERT_EQ(FOLLOWER, GetConsensusRole("B", cstate));
  ASSERT_EQ(FOLLOWER, GetConsensusRole("C", cstate));
  ASSERT_EQ(NON_PARTICIPANT, GetConsensusRole("D", cstate));
  cstate.set_leader_uuid("D");
  ASSERT_EQ(NON_PARTICIPANT, GetConsensusRole("D", cstate)); // Illegal.

  // Test GetParticipantRole() on the participants in the config.
  for (const auto& peer : config2.peers()) {
    ASSERT_EQ(GetParticipantRole(peer, cstate),
              GetConsensusRole(peer.permanent_uuid(), cstate));
  }
}

TEST(QuorumUtilTest, TestIsRaftConfigVoter) {
  RaftConfigPB config;
  AddPeer(&config, "A", V);
  AddPeer(&config, "B", N);
  AddPeer(&config, "C", U);

  // The case when membership type is not specified. That sort of configuration
  // would not pass VerifyRaftConfig(), though. Anyway, that should result
  // in non-voter since the member_type is initialized with UNKNOWN_MEMBER_TYPE.
  const string no_member_type_peer_uuid = "D";
  RaftPeerPB* no_member_type_peer = config.add_peers();
  no_member_type_peer->set_permanent_uuid(no_member_type_peer_uuid);
  no_member_type_peer->mutable_last_known_addr()->set_host(
      no_member_type_peer_uuid + ".example.com");

  ASSERT_TRUE(IsRaftConfigVoter("A", config));
  ASSERT_FALSE(IsRaftConfigVoter("B", config));
  ASSERT_FALSE(IsRaftConfigVoter("C", config));
  ASSERT_FALSE(IsRaftConfigVoter(no_member_type_peer_uuid, config));

  RaftPeerPB* peer_a;
  ASSERT_OK(GetRaftConfigMember(&config, "A", &peer_a));
  RaftPeerPB* peer_b;
  ASSERT_OK(GetRaftConfigMember(&config, "B", &peer_b));
  ASSERT_FALSE(ReplicaTypesEqual(*peer_a, *peer_b));
  ASSERT_TRUE(ReplicaTypesEqual(*peer_b, *peer_b));
  RaftPeerPB* peer_c;
  ASSERT_OK(GetRaftConfigMember(&config, "C", &peer_c));
  ASSERT_FALSE(ReplicaTypesEqual(*peer_b, *peer_c));
}

// Verify basic functionality of the kudu::consensus::ShouldAddReplica() utility
// function.
TEST(QuorumUtilTest, ShouldAddReplica) {
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V);
    AddPeer(&config, "B", V);
    AddPeer(&config, "C", V);
    EXPECT_FALSE(ShouldAddReplica(config, 2));
    EXPECT_FALSE(ShouldAddReplica(config, 3));
    // The configuration is under-replicated, but there are not enough healthy
    // voters to commit the configuration change.
    EXPECT_FALSE(ShouldAddReplica(config, 4));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '?');
    AddPeer(&config, "B", V, '?');
    AddPeer(&config, "C", V, '?');
    EXPECT_FALSE(ShouldAddReplica(config, 2));
    EXPECT_FALSE(ShouldAddReplica(config, 3));
    // The configuration is under-replicated, but there are not enough healthy
    // voters to commit the configuration change.
    EXPECT_FALSE(ShouldAddReplica(config, 4));
  }
  for (auto health_status : { '-', 'x' }) {
    SCOPED_TRACE(Substitute("health status '$0'", health_status));
    RaftConfigPB config;
    AddPeer(&config, "A", V, health_status);
    AddPeer(&config, "B", V, health_status);
    AddPeer(&config, "C", V, health_status);
    // The configuration is under-replicated, but there are not enough healthy
    // voters to commit the configuration change.
    EXPECT_FALSE(ShouldAddReplica(config, 4));
    EXPECT_FALSE(ShouldAddReplica(config, 2));
    EXPECT_FALSE(ShouldAddReplica(config, 3));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '?');
    AddPeer(&config, "B", V, '?');
    AddPeer(&config, "C", V, '-');
    EXPECT_FALSE(ShouldAddReplica(config, 2));
    // The configuration is under-replicated, but there are not enough healthy
    // voters to commit the configuration change.
    EXPECT_FALSE(ShouldAddReplica(config, 3));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", N, '+');
    EXPECT_FALSE(ShouldAddReplica(config, 2));
    EXPECT_TRUE(ShouldAddReplica(config, 3));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '?');
    AddPeer(&config, "B", V, '?');
    AddPeer(&config, "C", N, '+');
    EXPECT_FALSE(ShouldAddReplica(config, 2));
    // The configuration is under-replicated, but there are not enough healthy
    // voters to commit the configuration change.
    EXPECT_FALSE(ShouldAddReplica(config, 3));
  }
  for (auto health_status : { '-', 'x' }) {
    SCOPED_TRACE(Substitute("health status '$0'", health_status));
    RaftConfigPB config;
    AddPeer(&config, "A", V, '?');
    AddPeer(&config, "B", V, health_status);
    AddPeer(&config, "C", N, '+');
    // The configuration is over-replicated already.
    EXPECT_FALSE(ShouldAddReplica(config, 1));
    // Not enough voters to commit the change.
    EXPECT_FALSE(ShouldAddReplica(config, 2));
    EXPECT_FALSE(ShouldAddReplica(config, 3));
  }
  for (auto health_status : { '-', 'x' }) {
    SCOPED_TRACE(Substitute("health status '$0'", health_status));
    RaftConfigPB config;
    AddPeer(&config, "A", V, '?');
    AddPeer(&config, "B", V, health_status);
    AddPeer(&config, "C", N, '+', {{"PROMOTE", true}});
    EXPECT_FALSE(ShouldAddReplica(config, 1));
    EXPECT_FALSE(ShouldAddReplica(config, 2));
    // The configuration is under-replicated, but there are not enough healthy
    // voters to commit the configuration change.
    EXPECT_FALSE(ShouldAddReplica(config, 3));
  }
  for (auto health_status : { '-', 'x' }) {
    SCOPED_TRACE(Substitute("health status '$0'", health_status));
    RaftConfigPB config;
    AddPeer(&config, "A", V, '?');
    AddPeer(&config, "B", V, health_status);
    AddPeer(&config, "C", N, health_status, {{"PROMOTE", true}});
    EXPECT_FALSE(ShouldAddReplica(config, 1));
    EXPECT_FALSE(ShouldAddReplica(config, 2));
    EXPECT_FALSE(ShouldAddReplica(config, 3));
  }
  for (auto health_status : { '-', 'x' }) {
    SCOPED_TRACE(Substitute("health status '$0'", health_status));
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, health_status);
    EXPECT_FALSE(ShouldAddReplica(config, 2));
    EXPECT_TRUE(ShouldAddReplica(config, 3));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '?');
    EXPECT_FALSE(ShouldAddReplica(config, 2));
    // The catalog manager should wait for a definite health status of replica
    // 'C' before making decision whether to add replica for replacement or not.
    EXPECT_FALSE(ShouldAddReplica(config, 3));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '+', {{"REPLACE", true}});
    EXPECT_TRUE(ShouldAddReplica(config, 3));
    EXPECT_FALSE(ShouldAddReplica(config, 2));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "D", N, '+');
    EXPECT_TRUE(ShouldAddReplica(config, 4));
    EXPECT_TRUE(ShouldAddReplica(config, 3));
    EXPECT_FALSE(ShouldAddReplica(config, 2));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "D", N, '+', {{"PROMOTE", true}});
    EXPECT_TRUE(ShouldAddReplica(config, 4));
    EXPECT_FALSE(ShouldAddReplica(config, 3));
    EXPECT_FALSE(ShouldAddReplica(config, 2));
  }
  for (auto health_status : { '-', 'x' }) {
    SCOPED_TRACE(Substitute("health status '$0'", health_status));
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, health_status);
    AddPeer(&config, "D", N, health_status);
    EXPECT_FALSE(ShouldAddReplica(config, 2));
    EXPECT_TRUE(ShouldAddReplica(config, 3));
  }
  for (auto health_status : { '-', 'x' }) {
    SCOPED_TRACE(Substitute("health status '$0'", health_status));
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, health_status);
    AddPeer(&config, "D", N, '+');
    EXPECT_FALSE(ShouldAddReplica(config, 2));
    // The non-voter replica does not have the PROMOTE attribute,
    // so a new one is needed.
    EXPECT_TRUE(ShouldAddReplica(config, 3));
    EXPECT_TRUE(ShouldAddReplica(config, 4));
  }
  for (auto health_status : { '-', 'x' }) {
    SCOPED_TRACE(Substitute("health status '$0'", health_status));
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, health_status);
    AddPeer(&config, "D", N, '+', {{"PROMOTE", true}});
    EXPECT_FALSE(ShouldAddReplica(config, 2));
    EXPECT_FALSE(ShouldAddReplica(config, 3));
    EXPECT_TRUE(ShouldAddReplica(config, 4));
  }
  for (auto health_status : { '-', 'x' }) {
    SCOPED_TRACE(Substitute("health status '$0'", health_status));
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, health_status);
    AddPeer(&config, "D", N, health_status, {{"PROMOTE", true}});
    EXPECT_FALSE(ShouldAddReplica(config, 2));
    EXPECT_TRUE(ShouldAddReplica(config, 3));
  }
  for (auto health_status : { '-', 'x' }) {
    SCOPED_TRACE(Substitute("health status '$0'", health_status));
    RaftConfigPB config;
    AddPeer(&config, "A", V, health_status);
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '+');
    AddPeer(&config, "D", N, health_status, {{"PROMOTE", true}});
    AddPeer(&config, "E", N, '+', {{"PROMOTE", true}});
    EXPECT_FALSE(ShouldAddReplica(config, 3));
  }
  for (auto health_status : { '-', 'x' }) {
    SCOPED_TRACE(Substitute("health status '$0'", health_status));
    RaftConfigPB config;
    AddPeer(&config, "A", V, health_status);
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '+');
    AddPeer(&config, "D", N, health_status, {{"PROMOTE", true}});
    AddPeer(&config, "E", N, '+', {{"PROMOTE", false}});
    EXPECT_TRUE(ShouldAddReplica(config, 3));
  }
  for (auto health_status : { '-', 'x' }) {
    SCOPED_TRACE(Substitute("health status '$0'", health_status));
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", V, health_status);
    AddPeer(&config, "C", V, health_status);
    // If honoring the health of the replica's majority, the catalog manager
    // will not add a new non-voter replica until the situation is resolved.
    EXPECT_FALSE(ShouldAddReplica(config, 3));
  }
  for (auto health_status : { '-', 'x' }) {
    SCOPED_TRACE(Substitute("health status '$0'", health_status));
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", V, health_status);
    AddPeer(&config, "C", V, '+');
    AddPeer(&config, "D", V, health_status);
    AddPeer(&config, "E", V, '+');
    EXPECT_FALSE(ShouldAddReplica(config, 3));
    EXPECT_TRUE(ShouldAddReplica(config, 4));
    EXPECT_TRUE(ShouldAddReplica(config, 5));
  }
}

// Test that when tablet replicas are ignored for underreplication (e.g. due to
// maintenance mode of a tablet server), the decision to add a replica will
// actually ignore failures as appropriate.
TEST(QuorumUtilTest, ShouldAddReplicaIgnoreFailures) {
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '?');
    AddPeer(&config, "B", V, '-');
    AddPeer(&config, "C", V, '+');
    // The failed server is ignored, and doesn't count towards being
    // under-replicated. Note: The server with unknown health also doesn't
    // count towards being under-replicated.
    EXPECT_FALSE(ShouldAddReplica(config, 3, { "B" }));
    // While the server with unknown health doesn't count towards being
    // under-replicated, the failed server does. But since we require a
    // majority to add replicas, we can't add a replica.
    EXPECT_FALSE(ShouldAddReplica(config, 3, { "A" }));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '?');
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '+');
    // This is healthy, with or without ignoring failures.
    EXPECT_FALSE(ShouldAddReplica(config, 3));
    EXPECT_FALSE(ShouldAddReplica(config, 3, { "A" }));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '-');
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '+');
    // But when a healthy server is in maintenance mode, we should consider the
    // unhealthy server as failed and add a replica.
    EXPECT_TRUE(ShouldAddReplica(config, 3, { "B" }));
    // When the unhealthy server is in maintenance mode, we shouldn't add a
    // replica, since all three servers aren't considered failed.
    EXPECT_FALSE(ShouldAddReplica(config, 3, { "A" }));
    // And when everything is in maintenance mode, we shouldn't add a replica
    // even though a majority exists.
    EXPECT_FALSE(ShouldAddReplica(config, 3, { "A", "B", "C" }));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '-');
    AddPeer(&config, "B", V, '-');
    AddPeer(&config, "C", V, '+');
    // A majority doesn't exist, so no matter what failures are being ignored,
    // we will not add a replica.
    EXPECT_FALSE(ShouldAddReplica(config, 3, { "A" }));
    EXPECT_FALSE(ShouldAddReplica(config, 3, { "A", "B" }));
    EXPECT_FALSE(ShouldAddReplica(config, 3, { "A", "B", "C" }));
  }
  {
    // Ignored servers shouldn't change the decision when we really are
    // under-replicated.
    RaftConfigPB config;
    AddPeer(&config, "A", V, '-');
    AddPeer(&config, "B", V, '+');

    // No majority present.
    EXPECT_FALSE(ShouldAddReplica(config, 3, { "A" }));
    EXPECT_FALSE(ShouldAddReplica(config, 3, { "B" }));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '-');
    AddPeer(&config, "B", V, '-');
    AddPeer(&config, "C", V, '+');
    AddPeer(&config, "D", V, '+');
    AddPeer(&config, "E", V, '+');
    // When both failed replicas are being ignored, we shouldn't add a replica.
    EXPECT_FALSE(ShouldAddReplica(config, 5, { "A", "B" }));
    // When only one of them is ignored, we should.
    EXPECT_TRUE(ShouldAddReplica(config, 5, { "A" }));
  }
}

// Test that when tablet replicas are ignored for underreplication, replace is
// still honored as appropriate.
TEST(QuorumUtilTest, ShouldAddReplicaHonorReplaceWhenIgnoringFailures) {
  // Even if the replica to replace is meant to be ignored on failure, we
  // should honor the replacement and try to add a replica.
  for (char health : { '+', '-', '?' }) {
    RaftConfigPB config;
    AddPeer(&config, "A", V, health, {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '+');
    EXPECT_TRUE(ShouldAddReplica(config, 3, { "A" }));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "B", V, '-');
    AddPeer(&config, "C", V, '+');
    // Ignoring failures shouldn't impede our ability to add a replica when the
    // "ignored" server is actually healthy.
    EXPECT_TRUE(ShouldAddReplica(config, 3, { "A" }));
  }
}

TEST(QuorumUtilTest, ShouldAddReplicaHonorPromoteWhenIgnoringFailures) {
  // If one of our replicas to promote has failed, and we are supposed to
  // ignore its failure, we should not add a replica because of it.
  // And if they're healthy or unknown, we also shouldn't add a replica.
  for (char health : { '+', '-', '?' }) {
    {
      RaftConfigPB config;
      AddPeer(&config, "A", N, health, {{"PROMOTE", true}});
      AddPeer(&config, "B", V, '+');
      AddPeer(&config, "C", V, '+');
      EXPECT_FALSE(ShouldAddReplica(config, 3, { "A" }));
    }
    {
      RaftConfigPB config;
      AddPeer(&config, "A", N, health, {{"PROMOTE", true}});
      AddPeer(&config, "B", N, '-', {{"PROMOTE", true}});
      AddPeer(&config, "C", V, '+');
      AddPeer(&config, "D", V, '+');
      AddPeer(&config, "E", V, '+');
      EXPECT_FALSE(ShouldAddReplica(config, 5, { "A", "B" }));
      // But when there is a failure that isn't supposed to be ignored (B), we
      // should add a replica.
      EXPECT_TRUE(ShouldAddReplica(config, 5, { "A" }));
    }
  }
}

// Verify logic of the kudu::consensus::ShouldEvictReplica(), anticipating
// removal of a voter replica.
TEST(QuorumUtilTest, ShouldEvictReplicaVoters) {
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '?');
    AddPeer(&config, "B", V, '-');
    AddPeer(&config, "C", V, '+');
    // Not safe to evict because we don't have enough healthy nodes to commit
    // the eviction.
    EXPECT_FALSE(ShouldEvictReplica(config, "C", 1));
    EXPECT_FALSE(ShouldEvictReplica(config, "C", 2));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "B", V);
    AddPeer(&config, "C", V);
    EXPECT_FALSE(ShouldEvictReplica(config, "A", 3));
    EXPECT_FALSE(ShouldEvictReplica(config, "A", 2));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", V, '+', {{"REPLACE", false}});
    AddPeer(&config, "C", V, '-');
    AddPeer(&config, "D", V, '+');
    EXPECT_FALSE(ShouldEvictReplica(config, "A", 4));
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 3, &to_evict));
    EXPECT_EQ("C", to_evict);
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", V, '?');
    AddPeer(&config, "C", V, '+');
    AddPeer(&config, "D", V, '+');
    EXPECT_FALSE(ShouldEvictReplica(config, "A", 4));
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 3, &to_evict));
    EXPECT_EQ("B", to_evict);
  }
  for (char health_status : kHealthStatuses) {
    SCOPED_TRACE(Substitute("replica health status '$0'", health_status));
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '-');
    AddPeer(&config, "D", V, health_status, {{"REPLACE", true}});
    // For replication factors <= 3 we will be able to commit the eviction of D
    // with only A and B, regardless of D's health and regardless of the
    // desired replication factor.
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 2, &to_evict));
    // The priority of voter replica replacement (decreasing):
    //   * failed & slated for replacement
    //   * failed
    //   * ...
    if (health_status == '-' || health_status == 'x') {
      EXPECT_EQ("D", to_evict);
    } else {
      EXPECT_EQ("C", to_evict);
    }
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 3, &to_evict));
    if (health_status == '-' || health_status == 'x') {
      EXPECT_EQ("D", to_evict);
    } else {
      EXPECT_EQ("C", to_evict);
    }
    if (health_status == 'x') {
      // Unrecoverably failed replica should be evicted even if the configuration
      // is not over-replicated if it's safe to commit the configuration change.
      ASSERT_TRUE(ShouldEvictReplica(config, "A", 4));
      EXPECT_EQ("D", to_evict);
    } else {
      // Since we are not over-replicated, we will not evict in this case.
      EXPECT_FALSE(ShouldEvictReplica(config, "A", 4));
    }
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '?');
    AddPeer(&config, "D", V, '+', {{"REPLACE", true}});

    // For the replication factor 3, it's too early to evict 'C': it might be
    // in a good health, actually (reported, say, next heartbeat). Evicting 'D'
    // at this step is not a good idea neither: the 'C' might appear to fail,
    // and then it's better to keep 'D' around to provide the required
    // replication factor. It's necessary to wait for more deterministic status
    // of replica 'C' before making proper eviction decision.
    EXPECT_FALSE(ShouldEvictReplica(config, "A", 3));

    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 2, &to_evict));
    EXPECT_EQ("D", to_evict);
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", V, '?');
    AddPeer(&config, "C", V, '+');
    AddPeer(&config, "D", V, '+');
    EXPECT_FALSE(ShouldEvictReplica(config, "A", 4));
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 3, &to_evict));
    EXPECT_EQ("B", to_evict);
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", V, 'x');
    AddPeer(&config, "C", V, '+');
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 3, &to_evict));
    EXPECT_EQ("B", to_evict);
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 2, &to_evict));
    EXPECT_EQ("B", to_evict);
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", V, '-');
    AddPeer(&config, "C", V, 'x');

    // No majority to commit the change.
    EXPECT_FALSE(ShouldEvictReplica(config, "A", 3));
    EXPECT_FALSE(ShouldEvictReplica(config, "A", 2));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V);
    AddPeer(&config, "B", V);
    AddPeer(&config, "C", V);
    EXPECT_FALSE(ShouldEvictReplica(config, "", 2));
    EXPECT_FALSE(ShouldEvictReplica(config, "", 3));
  }
  for (auto health_status : { '-', 'x' }) {
    SCOPED_TRACE(Substitute("health status '$0'", health_status));
    RaftConfigPB config;
    AddPeer(&config, "A", V, '?');
    AddPeer(&config, "B", V, '?');
    AddPeer(&config, "C", V, health_status);
    EXPECT_FALSE(ShouldEvictReplica(config, "", 3));
    EXPECT_FALSE(ShouldEvictReplica(config, "", 2));
  }
  for (auto health_status : { '-', 'x' }) {
    SCOPED_TRACE(Substitute("health status '$0'", health_status));
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, health_status);
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 1, &to_evict));
    EXPECT_EQ("C", to_evict);
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 2));
    EXPECT_EQ("C", to_evict);
    if (health_status == '-') {
      EXPECT_FALSE(ShouldEvictReplica(config, "A", 3));
    } else {
      ASSERT_TRUE(ShouldEvictReplica(config, "A", 3, &to_evict));
      EXPECT_EQ("C", to_evict);
    }
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '?');
    AddPeer(&config, "B", V, '-');
    AddPeer(&config, "C", V, '+');
    EXPECT_FALSE(ShouldEvictReplica(config, "C", 3));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", V, '?');
    AddPeer(&config, "C", V, 'x');
    EXPECT_FALSE(ShouldEvictReplica(config, "A", 3));
  }
}

// Verify logic of the kudu::consensus::ShouldEvictReplica(), anticipating
// removal of a non-voter replica.
TEST(QuorumUtilTest, ShouldEvictReplicaNonVoters) {
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V);
    EXPECT_FALSE(ShouldEvictReplica(config, "", 1));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    EXPECT_FALSE(ShouldEvictReplica(config, "A", 1));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", N);
    EXPECT_FALSE(ShouldEvictReplica(config, "A", 2));
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 1, &to_evict));
    EXPECT_EQ("B", to_evict);
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", N, '+');
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 2, &to_evict));
    EXPECT_EQ("C", to_evict);
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 1, &to_evict));
    EXPECT_EQ("C", to_evict);
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", N, '-', {{"PROMOTE", true}});
    string to_evict;
    // It's always safe to evict an unhealthy non-voter if we have enough
    // healthy voters to commit the config change.
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 2, &to_evict));
    EXPECT_EQ("B", to_evict);
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 1, &to_evict));
    EXPECT_EQ("B", to_evict);
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", N, '-');
    AddPeer(&config, "C", N);
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 2, &to_evict));
    EXPECT_EQ("B", to_evict);
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 1, &to_evict));
    EXPECT_EQ("B", to_evict);
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", N, '?');
    AddPeer(&config, "C", N, '+');
    EXPECT_FALSE(ShouldEvictReplica(config, "A", 2));
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 1, &to_evict));
    EXPECT_EQ("B", to_evict);
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", N);
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 2, &to_evict));
    EXPECT_EQ("C", to_evict);
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 1, &to_evict));
    EXPECT_EQ("C", to_evict);
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", V);
    AddPeer(&config, "C", N);
    EXPECT_FALSE(ShouldEvictReplica(config, "A", 2));
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 1, &to_evict));
    // Would evict a non-voter first, but it's not known whether the majority
    // of the voter replicas are on-line to commence the operation: that's
    // because the state of B is unknown. So, in this case the voter replica B
    // will be removed first.
    EXPECT_EQ("B", to_evict);

    RemovePeer(&config, "B");
    // Now, having just a single online replica, it's possible to evict the
    // failed non-voter replica C.
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 1, &to_evict));
    EXPECT_EQ("C", to_evict);
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '-');
    AddPeer(&config, "B", V);
    AddPeer(&config, "C", N);
    EXPECT_FALSE(ShouldEvictReplica(config, "", 2));
    EXPECT_FALSE(ShouldEvictReplica(config, "", 1));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '-');
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '+');
    AddPeer(&config, "D", N, '+', {{"PROMOTE", true}});
    EXPECT_FALSE(ShouldEvictReplica(config, "B", 3));
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "B", 2, &to_evict));
    EXPECT_EQ("D", to_evict);
  }
  {
    // Make sure failed non-voter replicas are removed from the configuration to
    // avoid polluting all tablet servers with failed non-voter replicas.
    RaftConfigPB config;
    AddPeer(&config, "A", V, '-');
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '+');
    AddPeer(&config, "D", N, '-', {{"PROMOTE", true}});
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "B", 4, &to_evict));
    EXPECT_EQ("D", to_evict);
    ASSERT_TRUE(ShouldEvictReplica(config, "C", 3, &to_evict));
    EXPECT_EQ("D", to_evict);
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '-');
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '+');
    AddPeer(&config, "D", N, '?', {{"PROMOTE", true}});
    EXPECT_FALSE(ShouldEvictReplica(config, "B", 3));
    EXPECT_FALSE(ShouldEvictReplica(config, "B", 4));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '-');
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '+');
    AddPeer(&config, "D", N, 'x', {{"PROMOTE", true}});
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "B", 3, &to_evict));
    EXPECT_EQ("D", to_evict);
    ASSERT_TRUE(ShouldEvictReplica(config, "B", 4, &to_evict));
    EXPECT_EQ("D", to_evict);
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, 'x');
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '+');
    AddPeer(&config, "D", N, 'x');
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "B", 3, &to_evict));
    EXPECT_EQ("D", to_evict);
    ASSERT_TRUE(ShouldEvictReplica(config, "B", 4, &to_evict));
    EXPECT_EQ("D", to_evict);
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, 'x');
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '+');
    AddPeer(&config, "D", N, '-');
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "B", 3, &to_evict));
    EXPECT_EQ("A", to_evict);
    ASSERT_TRUE(ShouldEvictReplica(config, "B", 4, &to_evict));
    EXPECT_EQ("A", to_evict);
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", V, '-');
    AddPeer(&config, "C", N, '-', {{"PROMOTE", true}});
    EXPECT_FALSE(ShouldEvictReplica(config, "A", 2));
    // Would evict a non-voter first, but replica B is reported as failed and
    // the configuration does not have enough healthy voter replicas to have a
    // majority of votes. So, the voter replica B will be removed first.
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 1, &to_evict));
    EXPECT_EQ("B", to_evict);

    RemovePeer(&config, "B");
    // Now, having just a single online replica, it's possible to evict the
    // failed non-voter replica C.
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 1, &to_evict));
    EXPECT_EQ("C", to_evict);
  }
}

TEST(QuorumUtilTest, DontEvictLeader) {
  const vector<string> replicas = { "A", "B", "C", "D" };
  RaftConfigPB config;
  AddPeer(&config, replicas[0], V, '+');
  AddPeer(&config, replicas[1], V, '+');
  AddPeer(&config, replicas[2], V, '+');
  AddPeer(&config, replicas[3], V, '+');

  // Exhaustively loop through all nodes, each as leader, when over-replicated
  // and ensure that the leader never gets evicted.
  for (const auto& leader : replicas) {
    SCOPED_TRACE(Substitute("leader $0", leader));
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, leader, 3, &to_evict));
    ASSERT_NE(leader, to_evict);
  }
}

// This is a scenario for tablet configurations with more than the required
// number of voter replicas.
TEST(QuorumUtilTest, TooManyVoters) {
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '?');
    AddPeer(&config, "D", V, '-');
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 3, &to_evict));
    EXPECT_EQ("D", to_evict);
    EXPECT_FALSE(ShouldAddReplica(config, 3));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+');
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '-');
    AddPeer(&config, "D", V, '-');
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 3, &to_evict));
    EXPECT_TRUE(to_evict == "C" || to_evict == "D") << to_evict;
    EXPECT_FALSE(ShouldAddReplica(config, 3));
  }
}

// Basic scenarios involving replicas with the REPLACE attribute set.
TEST(QuorumUtilTest, ReplaceAttributeBasic) {
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    EXPECT_TRUE(ShouldAddReplica(config, 1));
    EXPECT_FALSE(ShouldEvictReplica(config, "A", 1));
  }
  {
    // Regression test scenario for KUDU-2443.
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+');
    EXPECT_FALSE(ShouldAddReplica(config, 1));
    EXPECT_FALSE(ShouldEvictReplica(config, "A", 1));
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "B", 1, &to_evict));
    EXPECT_EQ("A", to_evict);
  }
  {
    for (auto health_status : { '+', '-', '?', 'x' }) {
      SCOPED_TRACE(Substitute("health status '$0'", health_status));
      RaftConfigPB config;
      AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
      AddPeer(&config, "B", N, health_status);
      EXPECT_TRUE(ShouldAddReplica(config, 1));
      if (health_status == '+' || health_status == '?') {
        EXPECT_FALSE(ShouldEvictReplica(config, "A", 1));
      } else {
        string to_evict;
        ASSERT_TRUE(ShouldEvictReplica(config, "A", 1, &to_evict));
        EXPECT_EQ("B", to_evict);
      }
    }
  }
  // If a non-voter replica with PROMOTE=true is already in the Raft config,
  // no need to add an additional one if the health status of the non-voter
  // replica is HEALTHY or UNKNOWN.
  {
    for (auto health_status : { '+', '-', '?', 'x' }) {
      SCOPED_TRACE(Substitute("health status '$0'", health_status));
      RaftConfigPB config;
      AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
      AddPeer(&config, "B", N, health_status, {{"PROMOTE", true}});
      if (health_status == '+' || health_status == '?') {
        EXPECT_FALSE(ShouldAddReplica(config, 1));
      } else {
        EXPECT_TRUE(ShouldAddReplica(config, 1));
      }
      if (health_status == '+' || health_status == '?') {
        EXPECT_FALSE(ShouldEvictReplica(config, "A", 1));
      } else {
        string to_evict;
        ASSERT_TRUE(ShouldEvictReplica(config, "A", 1, &to_evict));
        EXPECT_EQ("B", to_evict);
      }
    }
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '+');
    EXPECT_FALSE(ShouldEvictReplica(config, "A", 3));
    EXPECT_TRUE(ShouldAddReplica(config, 3));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '+');
    AddPeer(&config, "D", V, '+');
    EXPECT_FALSE(ShouldEvictReplica(config, "A", 3));
    EXPECT_FALSE(ShouldAddReplica(config, 3));

    for (const auto& leader_replica : { "B", "C", "D" }) {
      string to_evict;
      SCOPED_TRACE(Substitute("leader $0", leader_replica));
      ASSERT_TRUE(ShouldEvictReplica(config, leader_replica, 3, &to_evict));
      EXPECT_EQ("A", to_evict);
    }
  }
  for (auto health_status : { '-', '?', 'x' }) {
    RaftConfigPB config;
    AddPeer(&config, "A", V, health_status, {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '+');
    AddPeer(&config, "D", V, '+');
    for (const auto& leader_replica : { "B", "C", "D" }) {
      SCOPED_TRACE(Substitute("health status '$0', leader $1",
                              health_status, leader_replica));
      string to_evict;
      ASSERT_TRUE(ShouldEvictReplica(config, leader_replica, 3, &to_evict));
      EXPECT_EQ("A", to_evict);
      EXPECT_FALSE(ShouldAddReplica(config, 3));
    }
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '+');
    AddPeer(&config, "D", V, '+');
    AddPeer(&config, "E", V, '+');
    // There should be no attempt to evict the leader.
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 3, &to_evict));
    EXPECT_NE("A", to_evict);
    EXPECT_FALSE(ShouldAddReplica(config, 3));

    for (const auto& leader_replica : { "B", "C", "D", "E" }) {
      string to_evict;
      SCOPED_TRACE(Substitute("leader $0", leader_replica));
      ASSERT_TRUE(ShouldEvictReplica(config, leader_replica, 3, &to_evict));
      EXPECT_EQ("A", to_evict);
    }
  }
  for (auto replica_health : kHealthStatuses) {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "C", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "D", V, replica_health, {{"REPLACE", true}});
    SCOPED_TRACE(Substitute("replica health status '$0'", replica_health));
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 3, &to_evict));
    if (replica_health == '+') {
      EXPECT_NE("A", to_evict);
    } else {
      EXPECT_EQ("D", to_evict);
    }
    EXPECT_TRUE(ShouldAddReplica(config, 3));
  }
  for (auto health_status : { '?', '-', 'x' }) {
    RaftConfigPB config;
    AddPeer(&config, "A", V, health_status, {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "C", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "D", V, '+');
    for (const auto& leader_replica : { "B", "C", "D" }) {
      SCOPED_TRACE(Substitute("health status '$0', leader $1",
                              health_status, leader_replica));
      string to_evict;
      ASSERT_TRUE(ShouldEvictReplica(config, leader_replica, 3, &to_evict));
      EXPECT_EQ("A", to_evict);
      EXPECT_TRUE(ShouldAddReplica(config, 3));
    }
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "C", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "D", V, '-');
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 3, &to_evict));
    EXPECT_EQ("D", to_evict);
    EXPECT_TRUE(ShouldAddReplica(config, 3));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "C", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "D", V, '?');
    EXPECT_FALSE(ShouldEvictReplica(config, "B", 3));
    EXPECT_TRUE(ShouldAddReplica(config, 3));
  }
  for (auto health_status : { '?', '-', 'x' }) {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "C", V, '?');
    AddPeer(&config, "D", V, health_status, {{"REPLACE", true}});
    AddPeer(&config, "E", V, '+');
    SCOPED_TRACE(Substitute("health status '$0'", health_status));
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 3, &to_evict));
    EXPECT_EQ("D", to_evict);
    EXPECT_TRUE(ShouldAddReplica(config, 3));
  }
}

// Test specific to the scenarios where the leader replica itself marked with
// the 'REPLACE' attribute.
TEST(QuorumUtilTest, LeaderReplicaWithReplaceAttribute) {
  // Healthy excess voter replicas (both voters and non-voters) should not be
  // evicted when the leader is marked with the 'REPLACE' attribute.
  for (auto health_status : { '+', '?' }) {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '+');
    AddPeer(&config, "D", N, health_status, {{"PROMOTE", true}});
    SCOPED_TRACE(Substitute("non-voter replica with status '$0'", health_status));
    EXPECT_FALSE(ShouldEvictReplica(config, "A", 3));
    EXPECT_FALSE(ShouldAddReplica(config, 3));
  }
  for (auto health_status : { '+', '?' }) {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '+');
    AddPeer(&config, "D", V, health_status);
    SCOPED_TRACE(Substitute("voter replica with status '$0'", health_status));
    EXPECT_FALSE(ShouldEvictReplica(config, "A", 3));
    EXPECT_FALSE(ShouldAddReplica(config, 3));
  }
  for (auto promote : { false, true }) {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '+');
    AddPeer(&config, "D", N, '-', {{"PROMOTE", promote}});
    SCOPED_TRACE(Substitute(
        "failed non-voter replica with PROMOTE attribute $0",
        promote ? "set" : "unset"));
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 3, &to_evict));
    EXPECT_EQ("D", to_evict);
    EXPECT_TRUE(ShouldAddReplica(config, 3));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '+');
    AddPeer(&config, "D", V, '-');
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 3, &to_evict));
    EXPECT_EQ("D", to_evict);
    EXPECT_TRUE(ShouldAddReplica(config, 3));
  }
  {
    // Current algorithm is conservative in the cases like below, but we might
    // evict non-voter replica 'E' which does not have the PROMOTE attribute.
    // TODO(aserbin): clarify on this.
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '+');
    AddPeer(&config, "D", N, '+', {{"PROMOTE", true}});
    AddPeer(&config, "E", N, '+');
    EXPECT_FALSE(ShouldEvictReplica(config, "A", 3));
    EXPECT_FALSE(ShouldAddReplica(config, 3));
  }
  {
    // The non-voter replica does not have the 'promote' attribute, so
    // it should be evicted since it's not going to become a voter anyway,
    // and we don't support standby non-voter replicas at this point.
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '+');
    AddPeer(&config, "D", V, '+');
    AddPeer(&config, "E", N, '+');
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 3, &to_evict));
    EXPECT_EQ("E", to_evict);
    EXPECT_FALSE(ShouldAddReplica(config, 3));
  }
  {
    // In the case below the non-voter replica 'D' is not needed. The
    // configuration like that might be the result of an attempt to replace 'D'
    // which was previously reported as failed. However, by the time the newly
    // added replica caught up with the leader, replica 'E' was back on-line.
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '+');
    AddPeer(&config, "D", N, '+', {{"PROMOTE", true}});
    AddPeer(&config, "E", V, '+');
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 3, &to_evict));
    EXPECT_EQ("D", to_evict);
    EXPECT_FALSE(ShouldAddReplica(config, 3));
  }
}

// This test is specific for various scenarios when multiple replicas have the
// REPLACE attribute set.
TEST(QuorumUtilTest, MultipleReplicasWithReplaceAttribute) {
  for (auto replica_type : { N, V }) {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "C", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "D", replica_type, '-');
    SCOPED_TRACE(Substitute("replica of $0 type",
                            RaftPeerPB::MemberType_Name(replica_type)));
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 3, &to_evict));
    EXPECT_EQ("D", to_evict);
    EXPECT_TRUE(ShouldAddReplica(config, 3));
  }
  for (auto replica_health : { '+', '?' }) {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "C", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "D", N, replica_health);
    SCOPED_TRACE(Substitute("NON_VOTER replica with health status '$0'",
                            replica_health));
    EXPECT_FALSE(ShouldEvictReplica(config, "A", 3));
    EXPECT_TRUE(ShouldAddReplica(config, 3));
  }
  for (const auto& leader_replica : { "A", "B", "C" }) {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "C", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "D", V, '?');
    SCOPED_TRACE(Substitute("leader $0", leader_replica));
    EXPECT_FALSE(ShouldEvictReplica(config, leader_replica, 3));
    EXPECT_TRUE(ShouldAddReplica(config, 3));
  }
  for (const auto& leader_replica : { "A", "C" }) {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "C", V, '+');
    AddPeer(&config, "D", N, '+', {{"PROMOTE", true}});
    AddPeer(&config, "E", N, '+', {{"PROMOTE", true}});
    SCOPED_TRACE(Substitute("leader $0", leader_replica));
    EXPECT_FALSE(ShouldEvictReplica(config, leader_replica, 3));
    EXPECT_FALSE(ShouldAddReplica(config, 3));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "C", V, '+', {{"REPLACE", true}});
    EXPECT_FALSE(ShouldEvictReplica(config, "A", 3));
    EXPECT_TRUE(ShouldAddReplica(config, 3));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "C", V, '+');
    AddPeer(&config, "D", V, '+');
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 3, &to_evict));
    EXPECT_EQ("B", to_evict);
    EXPECT_TRUE(ShouldAddReplica(config, 3));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "C", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "D", V, '+');
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 3, &to_evict));
    EXPECT_TRUE(to_evict == "B" || to_evict == "C");
    EXPECT_TRUE(ShouldAddReplica(config, 3));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '+');
    AddPeer(&config, "D", N, '+');
    EXPECT_FALSE(ShouldEvictReplica(config, "A", 3));
    // The non-voter replica does not have the PROMOTE attribute, so it the
    // configuration should be considered under-replicated.
    EXPECT_TRUE(ShouldAddReplica(config, 3));
  }
  for (auto replica_status : { '+', '?' }) {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '+');
    AddPeer(&config, "D", N, replica_status, {{"PROMOTE", true}});
    EXPECT_FALSE(ShouldEvictReplica(config, "A", 3));
    EXPECT_FALSE(ShouldAddReplica(config, 3));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+');
    AddPeer(&config, "C", V, '+');
    AddPeer(&config, "D", N, '-');
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 3, &to_evict));
    EXPECT_EQ("D", to_evict);
    EXPECT_TRUE(ShouldAddReplica(config, 3));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "C", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "D", V, '+');
    AddPeer(&config, "E", V, '+');
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "A", 3, &to_evict));
    EXPECT_TRUE(to_evict == "B" || to_evict == "C");
    EXPECT_TRUE(ShouldAddReplica(config, 3));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "C", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "D", V, '+');
    AddPeer(&config, "E", V, '+');
    AddPeer(&config, "F", V, '+');

    for (const string& leader_replica : { "A", "B", "C", "D", "E", "F" }) {
      string to_evict;
      ASSERT_TRUE(ShouldEvictReplica(config, leader_replica, 3, &to_evict));
      EXPECT_TRUE(to_evict == "A" || to_evict == "B" || to_evict == "C");
      if (leader_replica == "A" || leader_replica == "B" || leader_replica == "C") {
        EXPECT_NE(leader_replica, to_evict);
      }
    }
    EXPECT_FALSE(ShouldAddReplica(config, 3));
  }
  {
    RaftConfigPB config;
    AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "B", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "C", V, '+', {{"REPLACE", true}});
    AddPeer(&config, "D", N, '+', {{"PROMOTE", true}});
    AddPeer(&config, "E", N, '+', {{"PROMOTE", true}});
    AddPeer(&config, "F", N, '+', {{"PROMOTE", true}});

    for (const string& leader_replica : { "A", "B", "C" }) {
      // All non-voters are in good shape and not a single one has been
      // promoted yet.
      ASSERT_FALSE(ShouldEvictReplica(config, leader_replica, 3));
    }
    // No more replicas are needed for the replacement.
    EXPECT_FALSE(ShouldAddReplica(config, 3));
  }
}

// A scenario of replica replacement where replicas fall behind the log segment
// GC threshold and are replaced accordingly. This scenario is written to
// address scenarios like of KUDU-2342.
TEST(QuorumUtilTest, NewlyAddedNonVoterFallsBehindLogGC) {
  constexpr auto kReplicationFactor = 3;

  RaftConfigPB config;
  AddPeer(&config, "A", V, '+');
  AddPeer(&config, "B", V, '+');
  AddPeer(&config, "C", V, '+');

  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  // Replica B falls behind the log segment GC threshold. Since this is an
  // irreverisble failure, system tries to evict the replica right away.
  SetPeerHealth(&config, "B", 'x');
  string to_evict;
  ASSERT_TRUE(ShouldEvictReplica(config, "A", kReplicationFactor, &to_evict));
  EXPECT_EQ("B", to_evict);

  RemovePeer(&config, to_evict);
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_TRUE(ShouldAddReplica(config, kReplicationFactor));

  // Adding a non-voter to replace B.
  AddPeer(&config, "D", N, '?', {{"PROMOTE", true}});
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  // The new non-voter replica becomes healthy.
  SetPeerHealth(&config, "D", '+');
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  // The new non-voter replica falls behind the log segment GC threshold. The
  // system should evict it before trying to add a replacement replica.
  SetPeerHealth(&config, "D", 'x');
  ASSERT_TRUE(ShouldEvictReplica(config, "A", kReplicationFactor, &to_evict));
  EXPECT_EQ("D", to_evict);
  RemovePeer(&config, to_evict);

  // A new non-voter replica is needed.
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_TRUE(ShouldAddReplica(config, kReplicationFactor));

  // Adding a non-voter to replace D.
  AddPeer(&config, "E", N, '?', {{"PROMOTE", true}});
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  // The new non-voter replica 'E' becomes healthy.
  SetPeerHealth(&config, "E", '+');
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  // The newly added replica gets promoted to voter.
  PromotePeer(&config, "E");
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  // The new voter replica E falls behind the log segment GC threshold. The
  // replica should be evicted.
  SetPeerHealth(&config, "E", 'x');
  ASSERT_TRUE(ShouldEvictReplica(config, "A", kReplicationFactor, &to_evict));
  EXPECT_EQ("E", to_evict);

  RemovePeer(&config, to_evict);
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_TRUE(ShouldAddReplica(config, kReplicationFactor));

  // The system should add a replacement for the evicted replica.
  AddPeer(&config, "F", N, '?', {{"PROMOTE", true}});
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  // The new non-voter replica 'F' becomes healthy.
  SetPeerHealth(&config, "F", '+');
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  // The newly added replica 'F' gets promoted to voter, all is well now.
  PromotePeer(&config, "F");
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));
}

// A scenario of replica replacement where the replica added for replacement
// of a failed one also fails. The system should end up replacing both failed
// replicas.
TEST(QuorumUtilTest, NewlyPromotedReplicaCrashes) {
  constexpr auto kReplicationFactor = 3;

  RaftConfigPB config;
  AddPeer(&config, "A", V, '+');
  AddPeer(&config, "B", V, '+');
  AddPeer(&config, "C", V, '+');

  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  // Replica B fails.
  SetPeerHealth(&config, "B", '-');
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_TRUE(ShouldAddReplica(config, kReplicationFactor));

  // Adding a non-voter to replace B.
  AddPeer(&config, "D", N, '?', {{"PROMOTE", true}});
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  // The new non-voter replica becomes healthy.
  SetPeerHealth(&config, "D", '+');
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  // The newly added non-voter replica is promoted.
  PromotePeer(&config, "D");
  {
    // B would be evicted, if it's reported as is.
    string to_evict;
    ASSERT_TRUE(ShouldEvictReplica(config, "A", kReplicationFactor, &to_evict));
    EXPECT_EQ("B", to_evict);
  }
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  // However, the newly promoted replica crashes prior to B getting evicted.
  // The system should add a new replica for replacement.

  // We cannot evict because we don't have enough healthy voters to commit
  // the eviction config change.
  SetPeerHealth(&config, "D", '?');
  string to_evict;
  ASSERT_TRUE(ShouldEvictReplica(config, "A", kReplicationFactor, &to_evict));
  EXPECT_EQ("B", to_evict);
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  SetPeerHealth(&config, "D", '-');
  ASSERT_TRUE(ShouldEvictReplica(config, "A", kReplicationFactor, &to_evict));
  EXPECT_TRUE(to_evict == "B" || to_evict == "D") << to_evict;
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  RemovePeer(&config, to_evict);
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_TRUE(ShouldAddReplica(config, kReplicationFactor));

  AddPeer(&config, "E", N, '?', {{"PROMOTE", true}});
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  SetPeerHealth(&config, "E", '+');
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  PromotePeer(&config, "E");
  ASSERT_TRUE(ShouldEvictReplica(config, "A", kReplicationFactor, &to_evict));
  EXPECT_TRUE(to_evict == "B" || to_evict == "D") << to_evict;
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  RemovePeer(&config, to_evict);
  // The processs converges: 3 voter replicas, all are healthy.
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));
}

// A scenario to verify that the catalog manager does not do anything unexpected
// in the 3-4-3 replica management mode when replica's health is flapping
// between HEALTHY and UNKNOWN (e.g., when leader replica changes).
TEST(QuorumUtilTest, ReplicaHealthFlapping) {
  constexpr auto kReplicationFactor = 3;

  // The initial tablet report after the tablet replica A has started and
  // become the leader.
  RaftConfigPB config;
  AddPeer(&config, "A", V, '+');
  AddPeer(&config, "B", V, '?');
  AddPeer(&config, "C", V, '?');
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  // Replica B is reported as healthy.
  SetPeerHealth(&config, "B", '+');
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  // Replica C is reported as healthy.
  SetPeerHealth(&config, "C", '+');
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  // Replica B becomes the new leader.
  SetPeerHealth(&config, "A", '?');
  SetPeerHealth(&config, "B", '+');
  SetPeerHealth(&config, "C", '?');
  EXPECT_FALSE(ShouldEvictReplica(config, "B", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  // Replica A is reported as healthy; replica C fails.
  SetPeerHealth(&config, "A", '+');
  SetPeerHealth(&config, "B", '+');
  SetPeerHealth(&config, "C", '-');
  EXPECT_FALSE(ShouldEvictReplica(config, "B", kReplicationFactor));
  EXPECT_TRUE(ShouldAddReplica(config, kReplicationFactor));

  // A new non-voter replica has been added to replace failed replica C.
  AddPeer(&config, "D", N, '?', {{"PROMOTE", true}});
  EXPECT_FALSE(ShouldEvictReplica(config, "B", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  // Replica A becomes the new leader.
  SetPeerHealth(&config, "A", '+');
  SetPeerHealth(&config, "B", '?');
  SetPeerHealth(&config, "C", '?');
  SetPeerHealth(&config, "D", '?');
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  // The new leader has contacted on-line replicas.
  SetPeerHealth(&config, "A", '+');
  SetPeerHealth(&config, "B", '+');
  SetPeerHealth(&config, "C", '?');
  SetPeerHealth(&config, "D", '+');
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  // Replica D catches up with the leader's WAL and gets promoted.
  PromotePeer(&config, "D");
  string to_evict;
  ASSERT_TRUE(ShouldEvictReplica(config, "A", kReplicationFactor, &to_evict));
  EXPECT_EQ("C", to_evict);
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  // Replica D becomes the new leader.
  SetPeerHealth(&config, "A", '?');
  SetPeerHealth(&config, "B", '?');
  SetPeerHealth(&config, "C", '?');
  SetPeerHealth(&config, "D", '+');
  EXPECT_FALSE(ShouldEvictReplica(config, "D", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  SetPeerHealth(&config, "A", '+');
  SetPeerHealth(&config, "B", '+');
  SetPeerHealth(&config, "C", '?');
  SetPeerHealth(&config, "D", '+');
  ASSERT_TRUE(ShouldEvictReplica(config, "D", kReplicationFactor, &to_evict));
  EXPECT_EQ("C", to_evict);
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  SetPeerHealth(&config, "C", '-');
  ASSERT_TRUE(ShouldEvictReplica(config, "D", kReplicationFactor, &to_evict));
  EXPECT_EQ("C", to_evict);
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  RemovePeer(&config, "C");
  EXPECT_FALSE(ShouldEvictReplica(config, "D", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));
}

// A scenario to simulate the process of migrating all replicas of a tablet,
// where all replicas are marked for replacement simultaneously. This is a
// possible scenario when decommissioning multiple tablet servers/nodes at once.
TEST(QuorumUtilTest, ReplaceAllTabletReplicas) {
  constexpr auto kReplicationFactor = 3;

  // The initial tablet report after the tablet replica 'A' has started and
  // become the leader.
  RaftConfigPB config;
  AddPeer(&config, "A", V, '+', {{"REPLACE", true}});
  AddPeer(&config, "B", V, '+', {{"REPLACE", true}});
  AddPeer(&config, "C", V, '+', {{"REPLACE", true}});
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_TRUE(ShouldAddReplica(config, kReplicationFactor));

  // First non-voter replica added.
  AddPeer(&config, "D", N, '?', {{"PROMOTE", true}});
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_TRUE(ShouldAddReplica(config, kReplicationFactor));

  // Second non-voter replica added.
  AddPeer(&config, "E", N, '?', {{"PROMOTE", true}});
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_TRUE(ShouldAddReplica(config, kReplicationFactor));

  // Third non-voter replica added.
  AddPeer(&config, "F", N, '?', {{"PROMOTE", true}});
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  SetPeerHealth(&config, "D", '+');
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  // Replica 'D' catches up with the leader's WAL and gets promoted.
  PromotePeer(&config, "D");
  string to_evict;
  ASSERT_TRUE(ShouldEvictReplica(config, "A", kReplicationFactor, &to_evict));
  EXPECT_TRUE(to_evict == "B" || to_evict == "C");
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  // Evicting the replica selected by ShouldEvictReplica() above.
  RemovePeer(&config, to_evict);
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  // Non-voter replica 'F' become unavailable.
  SetPeerHealth(&config, "F", '-');
  ASSERT_TRUE(ShouldEvictReplica(config, "A", kReplicationFactor, &to_evict));
  ASSERT_EQ("F", to_evict);
  EXPECT_TRUE(ShouldAddReplica(config, kReplicationFactor));

  // Evicting the failed non-voter replica, selected by ShouldEvictReplica() above.
  RemovePeer(&config, to_evict);
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_TRUE(ShouldAddReplica(config, kReplicationFactor));

  // Adding a new non-voter replica.
  AddPeer(&config, "G", N, '?', {{"PROMOTE", true}});
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  // A newly added non-voter replica is in good shape.
  SetPeerHealth(&config, "G", '+');
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  // Replica 'E' is reported in good health.
  SetPeerHealth(&config, "E", '+');
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  // Replica 'E' catches up with the leader's WAL and gets promoted.
  PromotePeer(&config, "E");
  ASSERT_TRUE(ShouldEvictReplica(config, "A", kReplicationFactor, &to_evict));
  EXPECT_TRUE(to_evict == "B" || to_evict == "C");
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  // Evicting the replica selected by ShouldEvictReplica() above.
  RemovePeer(&config, to_evict);
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  // Replica 'G' catches up, but replica 'A' cannot yet be evicted since it's
  // a leader replica.
  PromotePeer(&config, "G");
  EXPECT_FALSE(ShouldEvictReplica(config, "A", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  // Leadership changes from 'A' to 'G', so now it's possible to evict 'A'.
  ASSERT_TRUE(ShouldEvictReplica(config, "G", kReplicationFactor, &to_evict));
  ASSERT_EQ("A", to_evict);
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));

  // Evicting the replica selected by ShouldEvictReplica() above. With that,
  // the replacement process of all the marked replicas is complete; no further
  // changes is necessary for the tablet's Raft configuration.
  RemovePeer(&config, to_evict);
  EXPECT_FALSE(ShouldEvictReplica(config, "G", kReplicationFactor));
  EXPECT_FALSE(ShouldAddReplica(config, kReplicationFactor));
}

} // namespace consensus
} // namespace kudu