hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java - hbase - Git at Google

 /**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
  * regarding copyright ownership.  The ASF licenses this file
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package org.apache.hadoop.hbase.rsgroup;

 import static org.junit.Assert.assertTrue;

 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.regex.Pattern;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.ClusterMetrics;
 import org.apache.hadoop.hbase.ClusterMetrics.Option;
 import org.apache.hadoop.hbase.HBaseCluster;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.NamespaceDescriptor;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.Waiter;
 import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.client.RegionInfo;
 import org.apache.hadoop.hbase.client.TableDescriptor;
 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
 import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
 import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
 import org.apache.hadoop.hbase.coprocessor.MasterObserver;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
 import org.apache.hadoop.hbase.master.ServerManager;
 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
 import org.apache.hadoop.hbase.net.Address;
 import org.apache.hadoop.hbase.quotas.QuotaUtil;
 import org.junit.Rule;
 import org.junit.rules.TestName;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

 import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
 import org.apache.hbase.thirdparty.com.google.common.collect.Sets;

 public abstract class TestRSGroupsBase {
   protected static final Logger LOG = LoggerFactory.getLogger(TestRSGroupsBase.class);

   // shared
   protected static final String GROUP_PREFIX = "Group";
   protected static final String TABLE_PREFIX = "Group";

   // shared, cluster type specific
   protected static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   protected static Admin ADMIN;
   protected static HBaseCluster CLUSTER;
   protected static HMaster MASTER;
   protected boolean INIT = false;
   protected static CPMasterObserver OBSERVER;

   public final static long WAIT_TIMEOUT = 60000;
   public final static int NUM_SLAVES_BASE = 4; // number of slaves for the smallest cluster
   public static int NUM_DEAD_SERVERS = 0;

   // Per test variables
   @Rule
   public TestName name = new TestName();
   protected TableName tableName;

   public static String getNameWithoutIndex(String name) {
     return name.split("\\[")[0];
   }

   public static void setUpTestBeforeClass() throws Exception {
     Configuration conf = TEST_UTIL.getConfiguration();
     conf.setFloat("hbase.master.balancer.stochastic.tableSkewCost", 6000);
     if (conf.get(RSGroupUtil.RS_GROUP_ENABLED) == null) {
       RSGroupUtil.enableRSGroup(conf);
     }
     if (conf.get(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY) != null) {
       conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
         conf.get(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY) + "," +
           CPMasterObserver.class.getName());
     } else {
       conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, CPMasterObserver.class.getName());
     }

     conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART,
       NUM_SLAVES_BASE - 1);
     conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
     conf.setInt("hbase.rpc.timeout", 100000);

     TEST_UTIL.startMiniCluster(NUM_SLAVES_BASE - 1);
     initialize();
   }

   protected static void initialize() throws Exception {
     ADMIN = new VerifyingRSGroupAdmin(TEST_UTIL.getConfiguration());
     CLUSTER = TEST_UTIL.getHBaseCluster();
     MASTER = TEST_UTIL.getMiniHBaseCluster().getMaster();

     // wait for balancer to come online
     TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
       @Override
       public boolean evaluate() throws Exception {
         return MASTER.isInitialized() &&
           ((RSGroupBasedLoadBalancer) MASTER.getLoadBalancer()).isOnline();
       }
     });
     ADMIN.balancerSwitch(false, true);
     MasterCoprocessorHost host = MASTER.getMasterCoprocessorHost();
     OBSERVER = (CPMasterObserver) host.findCoprocessor(CPMasterObserver.class.getName());
   }

   public static void tearDownAfterClass() throws Exception {
     TEST_UTIL.shutdownMiniCluster();
   }

   public void setUpBeforeMethod() throws Exception {
     LOG.info(name.getMethodName());
     tableName = TableName.valueOf(TABLE_PREFIX + "_" + name.getMethodName().split("\\[")[0]);
     if (!INIT) {
       INIT = true;
       tearDownAfterMethod();
     }
     OBSERVER.resetFlags();
   }

   public void tearDownAfterMethod() throws Exception {
     deleteTableIfNecessary();
     deleteNamespaceIfNecessary();
     deleteGroups();

     for (ServerName sn : ADMIN.listDecommissionedRegionServers()) {
       ADMIN.recommissionRegionServer(sn, null);
     }
     assertTrue(ADMIN.listDecommissionedRegionServers().isEmpty());

     int missing = NUM_SLAVES_BASE - getNumServers();
     LOG.info("Restoring servers: " + missing);
     for (int i = 0; i < missing; i++) {
       ((MiniHBaseCluster) CLUSTER).startRegionServer();
     }
     ADMIN.addRSGroup("master");
     ServerName masterServerName = ((MiniHBaseCluster) CLUSTER).getMaster().getServerName();
     try {
       ADMIN.moveServersToRSGroup(Sets.newHashSet(masterServerName.getAddress()), "master");
     } catch (Exception ex) {
       LOG.warn("Got this on setup, FYI", ex);
     }
     assertTrue(OBSERVER.preMoveServersCalled);
     TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
       @Override
       public boolean evaluate() throws Exception {
         LOG.info("Waiting for cleanup to finish " + ADMIN.listRSGroups());
         // Might be greater since moving servers back to default
         // is after starting a server

         return ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP).getServers().size() == NUM_SLAVES_BASE;
       }
     });
   }

   protected final RSGroupInfo addGroup(String groupName, int serverCount)
     throws IOException, InterruptedException {
     RSGroupInfo defaultInfo = ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP);
     ADMIN.addRSGroup(groupName);
     Set<Address> set = new HashSet<>();
     for (Address server : defaultInfo.getServers()) {
       if (set.size() == serverCount) {
         break;
       }
       set.add(server);
     }
     ADMIN.moveServersToRSGroup(set, groupName);
     RSGroupInfo result = ADMIN.getRSGroup(groupName);
     return result;
   }

   protected final void removeGroup(String groupName) throws IOException {
     Set<TableName> tables = new HashSet<>();
     for (TableDescriptor td : ADMIN.listTableDescriptors(true)) {
       RSGroupInfo groupInfo = ADMIN.getRSGroup(td.getTableName());
       if (groupInfo != null && groupInfo.getName().equals(groupName)) {
         tables.add(td.getTableName());
       }
     }
     ADMIN.setRSGroup(tables, RSGroupInfo.DEFAULT_GROUP);
     for (NamespaceDescriptor nd : ADMIN.listNamespaceDescriptors()) {
       if (groupName.equals(nd.getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP))) {
         nd.removeConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP);
         ADMIN.modifyNamespace(nd);
       }
     }
     RSGroupInfo groupInfo = ADMIN.getRSGroup(groupName);
     ADMIN.moveServersToRSGroup(groupInfo.getServers(), RSGroupInfo.DEFAULT_GROUP);
     ADMIN.removeRSGroup(groupName);
   }

   protected final void deleteTableIfNecessary() throws IOException {
     for (TableDescriptor desc : TEST_UTIL.getAdmin()
       .listTableDescriptors(Pattern.compile(TABLE_PREFIX + ".*"))) {
       TEST_UTIL.deleteTable(desc.getTableName());
     }
   }

   protected final void deleteNamespaceIfNecessary() throws IOException {
     for (NamespaceDescriptor desc : TEST_UTIL.getAdmin().listNamespaceDescriptors()) {
       if (desc.getName().startsWith(TABLE_PREFIX)) {
         ADMIN.deleteNamespace(desc.getName());
       }
     }
   }

   protected final void deleteGroups() throws IOException {
     for (RSGroupInfo groupInfo : ADMIN.listRSGroups()) {
       if (!groupInfo.getName().equals(RSGroupInfo.DEFAULT_GROUP)) {
         removeGroup(groupInfo.getName());
       }
     }
   }

   protected Map<TableName, List<String>> getTableRegionMap() throws IOException {
     Map<TableName, List<String>> map = Maps.newTreeMap();
     Map<TableName, Map<ServerName, List<String>>> tableServerRegionMap = getTableServerRegionMap();
     for (TableName tableName : tableServerRegionMap.keySet()) {
       if (!map.containsKey(tableName)) {
         map.put(tableName, new LinkedList<>());
       }
       for (List<String> subset : tableServerRegionMap.get(tableName).values()) {
         map.get(tableName).addAll(subset);
       }
     }
     return map;
   }

   protected Map<TableName, Map<ServerName, List<String>>> getTableServerRegionMap()
     throws IOException {
     Map<TableName, Map<ServerName, List<String>>> map = Maps.newTreeMap();
     Admin admin = TEST_UTIL.getAdmin();
     ClusterMetrics metrics =
       admin.getClusterMetrics(EnumSet.of(ClusterMetrics.Option.SERVERS_NAME));
     for (ServerName serverName : metrics.getServersName()) {
       for (RegionInfo region : admin.getRegions(serverName)) {
         TableName tableName = region.getTable();
         map.computeIfAbsent(tableName, k -> new TreeMap<>())
           .computeIfAbsent(serverName, k -> new ArrayList<>()).add(region.getRegionNameAsString());
       }
     }
     return map;
   }

   // return the real number of region servers, excluding the master embedded region server in 2.0+
   protected int getNumServers() throws IOException {
     ClusterMetrics status = ADMIN.getClusterMetrics(EnumSet.of(Option.MASTER, Option.LIVE_SERVERS));
     ServerName masterName = status.getMasterName();
     int count = 0;
     for (ServerName sn : status.getLiveServerMetrics().keySet()) {
       if (!sn.equals(masterName)) {
         count++;
       }
     }
     return count;
   }

   protected final String getGroupName(String baseName) {
     return GROUP_PREFIX + "_" + getNameWithoutIndex(baseName) + "_" +
       ThreadLocalRandom.current().nextInt(Integer.MAX_VALUE);
   }

   /**
    * The server name in group does not contain the start code, this method will find out the start
    * code and construct the ServerName object.
    */
   protected final ServerName getServerName(Address addr) {
     return TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().stream()
       .map(t -> t.getRegionServer().getServerName()).filter(sn -> sn.getAddress().equals(addr))
       .findFirst().get();
   }

   protected final void toggleQuotaCheckAndRestartMiniCluster(boolean enable) throws Exception {
     TEST_UTIL.shutdownMiniCluster();
     TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, enable);
     TEST_UTIL.startMiniCluster(NUM_SLAVES_BASE - 1);
     TEST_UTIL.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART,
       NUM_SLAVES_BASE - 1);
     TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
     initialize();
   }

   public static class CPMasterObserver implements MasterCoprocessor, MasterObserver {
     boolean preBalanceRSGroupCalled = false;
     boolean postBalanceRSGroupCalled = false;
     boolean preMoveServersCalled = false;
     boolean postMoveServersCalled = false;
     boolean preMoveTablesCalled = false;
     boolean postMoveTablesCalled = false;
     boolean preAddRSGroupCalled = false;
     boolean postAddRSGroupCalled = false;
     boolean preRemoveRSGroupCalled = false;
     boolean postRemoveRSGroupCalled = false;
     boolean preRemoveServersCalled = false;
     boolean postRemoveServersCalled = false;
     boolean preMoveServersAndTables = false;
     boolean postMoveServersAndTables = false;
     boolean preGetRSGroupInfoCalled = false;
     boolean postGetRSGroupInfoCalled = false;
     boolean preGetRSGroupInfoOfTableCalled = false;
     boolean postGetRSGroupInfoOfTableCalled = false;
     boolean preListRSGroupsCalled = false;
     boolean postListRSGroupsCalled = false;
     boolean preGetRSGroupInfoOfServerCalled = false;
     boolean postGetRSGroupInfoOfServerCalled = false;
     boolean preSetRSGroupForTablesCalled = false;
     boolean postSetRSGroupForTablesCalled = false;
     boolean preListTablesInRSGroupCalled = false;
     boolean postListTablesInRSGroupCalled = false;
     boolean preGetConfiguredNamespacesAndTablesInRSGroupCalled = false;
     boolean postGetConfiguredNamespacesAndTablesInRSGroupCalled = false;
     boolean preRenameRSGroup = false;
     boolean postRenameRSGroup = false;

     public void resetFlags() {
       preBalanceRSGroupCalled = false;
       postBalanceRSGroupCalled = false;
       preMoveServersCalled = false;
       postMoveServersCalled = false;
       preMoveTablesCalled = false;
       postMoveTablesCalled = false;
       preAddRSGroupCalled = false;
       postAddRSGroupCalled = false;
       preRemoveRSGroupCalled = false;
       postRemoveRSGroupCalled = false;
       preRemoveServersCalled = false;
       postRemoveServersCalled = false;
       preMoveServersAndTables = false;
       postMoveServersAndTables = false;
       preGetRSGroupInfoCalled = false;
       postGetRSGroupInfoCalled = false;
       preGetRSGroupInfoOfTableCalled = false;
       postGetRSGroupInfoOfTableCalled = false;
       preListRSGroupsCalled = false;
       postListRSGroupsCalled = false;
       preGetRSGroupInfoOfServerCalled = false;
       postGetRSGroupInfoOfServerCalled = false;
       preSetRSGroupForTablesCalled = false;
       postSetRSGroupForTablesCalled = false;
       preListTablesInRSGroupCalled = false;
       postListTablesInRSGroupCalled = false;
       preGetConfiguredNamespacesAndTablesInRSGroupCalled = false;
       postGetConfiguredNamespacesAndTablesInRSGroupCalled = false;
       preRenameRSGroup = false;
       postRenameRSGroup = false;
     }

     @Override
     public Optional<MasterObserver> getMasterObserver() {
       return Optional.of(this);
     }

     @Override
     public void preMoveServersAndTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
       Set<Address> servers, Set<TableName> tables, String targetGroup) throws IOException {
       preMoveServersAndTables = true;
     }

     @Override
     public void postMoveServersAndTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
       Set<Address> servers, Set<TableName> tables, String targetGroup) throws IOException {
       postMoveServersAndTables = true;
     }

     @Override
     public void preRemoveServers(final ObserverContext<MasterCoprocessorEnvironment> ctx,
       Set<Address> servers) throws IOException {
       preRemoveServersCalled = true;
     }

     @Override
     public void postRemoveServers(final ObserverContext<MasterCoprocessorEnvironment> ctx,
       Set<Address> servers) throws IOException {
       postRemoveServersCalled = true;
     }

     @Override
     public void preRemoveRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
       String name) throws IOException {
       preRemoveRSGroupCalled = true;
     }

     @Override
     public void postRemoveRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
       String name) throws IOException {
       postRemoveRSGroupCalled = true;
     }

     @Override
     public void preAddRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx, String name)
       throws IOException {
       preAddRSGroupCalled = true;
     }

     @Override
     public void postAddRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx, String name)
       throws IOException {
       postAddRSGroupCalled = true;
     }

     @Override
     public void preMoveTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
       Set<TableName> tables, String targetGroup) throws IOException {
       preMoveTablesCalled = true;
     }

     @Override
     public void postMoveTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
       Set<TableName> tables, String targetGroup) throws IOException {
       postMoveTablesCalled = true;
     }

     @Override
     public void preMoveServers(final ObserverContext<MasterCoprocessorEnvironment> ctx,
       Set<Address> servers, String targetGroup) throws IOException {
       preMoveServersCalled = true;
     }

     @Override
     public void postMoveServers(final ObserverContext<MasterCoprocessorEnvironment> ctx,
       Set<Address> servers, String targetGroup) throws IOException {
       postMoveServersCalled = true;
     }

     @Override
     public void preBalanceRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
       String groupName) throws IOException {
       preBalanceRSGroupCalled = true;
     }

     @Override
     public void postBalanceRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
       String groupName, boolean balancerRan) throws IOException {
       postBalanceRSGroupCalled = true;
     }

     @Override
     public void preGetRSGroupInfo(final ObserverContext<MasterCoprocessorEnvironment> ctx,
       final String groupName) throws IOException {
       preGetRSGroupInfoCalled = true;
     }

     @Override
     public void postGetRSGroupInfo(final ObserverContext<MasterCoprocessorEnvironment> ctx,
       final String groupName) throws IOException {
       postGetRSGroupInfoCalled = true;
     }

     @Override
     public void preGetRSGroupInfoOfTable(final ObserverContext<MasterCoprocessorEnvironment> ctx,
       final TableName tableName) throws IOException {
       preGetRSGroupInfoOfTableCalled = true;
     }

     @Override
     public void postGetRSGroupInfoOfTable(final ObserverContext<MasterCoprocessorEnvironment> ctx,
       final TableName tableName) throws IOException {
       postGetRSGroupInfoOfTableCalled = true;
     }

     @Override
     public void preListRSGroups(final ObserverContext<MasterCoprocessorEnvironment> ctx)
       throws IOException {
       preListRSGroupsCalled = true;
     }

     @Override
     public void postListRSGroups(final ObserverContext<MasterCoprocessorEnvironment> ctx)
       throws IOException {
       postListRSGroupsCalled = true;
     }

     @Override
     public void preGetRSGroupInfoOfServer(final ObserverContext<MasterCoprocessorEnvironment> ctx,
       final Address server) throws IOException {
       preGetRSGroupInfoOfServerCalled = true;
     }

     @Override
     public void postGetRSGroupInfoOfServer(final ObserverContext<MasterCoprocessorEnvironment> ctx,
       final Address server) throws IOException {
       postGetRSGroupInfoOfServerCalled = true;
     }

     @Override
     public void preListTablesInRSGroup(ObserverContext<MasterCoprocessorEnvironment> ctx,
       String groupName) throws IOException {
       preListTablesInRSGroupCalled = true;
     }

     @Override
     public void postListTablesInRSGroup(ObserverContext<MasterCoprocessorEnvironment> ctx,
       String groupName) throws IOException {
       postListTablesInRSGroupCalled = true;
     }

     @Override
     public void preGetConfiguredNamespacesAndTablesInRSGroup(
       ObserverContext<MasterCoprocessorEnvironment> ctx, String groupName) throws IOException {
       preGetConfiguredNamespacesAndTablesInRSGroupCalled = true;
     }

     @Override
     public void postGetConfiguredNamespacesAndTablesInRSGroup(
       ObserverContext<MasterCoprocessorEnvironment> ctx, String groupName) throws IOException {
       postGetConfiguredNamespacesAndTablesInRSGroupCalled = true;
     }

     @Override
     public void preRenameRSGroup(ObserverContext<MasterCoprocessorEnvironment> ctx, String oldName,
       String newName) throws IOException {
       preRenameRSGroup = true;
     }

     @Override
     public void postRenameRSGroup(ObserverContext<MasterCoprocessorEnvironment> ctx, String oldName,
       String newName) throws IOException {
       postRenameRSGroup = true;
     }
   }
 }
	/**
	* Licensed to the Apache Software Foundation (ASF) under one
	* or more contributor license agreements. See the NOTICE file
	* distributed with this work for additional information
	* regarding copyright ownership. The ASF licenses this file
	* to you under the Apache License, Version 2.0 (the
	* "License"); you may not use this file except in compliance
	* with the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/
	package org.apache.hadoop.hbase.rsgroup;

	import static org.junit.Assert.assertTrue;

	import java.io.IOException;
	import java.util.ArrayList;
	import java.util.EnumSet;
	import java.util.HashSet;
	import java.util.LinkedList;
	import java.util.List;
	import java.util.Map;
	import java.util.Optional;
	import java.util.Set;
	import java.util.TreeMap;
	import java.util.concurrent.ThreadLocalRandom;
	import java.util.regex.Pattern;
	import org.apache.hadoop.conf.Configuration;
	import org.apache.hadoop.hbase.ClusterMetrics;
	import org.apache.hadoop.hbase.ClusterMetrics.Option;
	import org.apache.hadoop.hbase.HBaseCluster;
	import org.apache.hadoop.hbase.HBaseTestingUtility;
	import org.apache.hadoop.hbase.MiniHBaseCluster;
	import org.apache.hadoop.hbase.NamespaceDescriptor;
	import org.apache.hadoop.hbase.ServerName;
	import org.apache.hadoop.hbase.TableName;
	import org.apache.hadoop.hbase.Waiter;
	import org.apache.hadoop.hbase.client.Admin;
	import org.apache.hadoop.hbase.client.RegionInfo;
	import org.apache.hadoop.hbase.client.TableDescriptor;
	import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
	import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
	import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
	import org.apache.hadoop.hbase.coprocessor.MasterObserver;
	import org.apache.hadoop.hbase.coprocessor.ObserverContext;
	import org.apache.hadoop.hbase.master.HMaster;
	import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
	import org.apache.hadoop.hbase.master.ServerManager;
	import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
	import org.apache.hadoop.hbase.net.Address;
	import org.apache.hadoop.hbase.quotas.QuotaUtil;
	import org.junit.Rule;
	import org.junit.rules.TestName;
	import org.slf4j.Logger;
	import org.slf4j.LoggerFactory;

	import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
	import org.apache.hbase.thirdparty.com.google.common.collect.Sets;

	public abstract class TestRSGroupsBase {
	protected static final Logger LOG = LoggerFactory.getLogger(TestRSGroupsBase.class);

	// shared
	protected static final String GROUP_PREFIX = "Group";
	protected static final String TABLE_PREFIX = "Group";

	// shared, cluster type specific
	protected static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
	protected static Admin ADMIN;
	protected static HBaseCluster CLUSTER;
	protected static HMaster MASTER;
	protected boolean INIT = false;
	protected static CPMasterObserver OBSERVER;

	public final static long WAIT_TIMEOUT = 60000;
	public final static int NUM_SLAVES_BASE = 4; // number of slaves for the smallest cluster
	public static int NUM_DEAD_SERVERS = 0;

	// Per test variables
	@Rule
	public TestName name = new TestName();
	protected TableName tableName;

	public static String getNameWithoutIndex(String name) {
	return name.split("\\[")[0];
	}

	public static void setUpTestBeforeClass() throws Exception {
	Configuration conf = TEST_UTIL.getConfiguration();
	conf.setFloat("hbase.master.balancer.stochastic.tableSkewCost", 6000);
	if (conf.get(RSGroupUtil.RS_GROUP_ENABLED) == null) {
	RSGroupUtil.enableRSGroup(conf);
	}
	if (conf.get(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY) != null) {
	conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
	conf.get(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY) + "," +
	CPMasterObserver.class.getName());
	} else {
	conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, CPMasterObserver.class.getName());
	}

	conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART,
	NUM_SLAVES_BASE - 1);
	conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
	conf.setInt("hbase.rpc.timeout", 100000);

	TEST_UTIL.startMiniCluster(NUM_SLAVES_BASE - 1);
	initialize();
	}

	protected static void initialize() throws Exception {
	ADMIN = new VerifyingRSGroupAdmin(TEST_UTIL.getConfiguration());
	CLUSTER = TEST_UTIL.getHBaseCluster();
	MASTER = TEST_UTIL.getMiniHBaseCluster().getMaster();

	// wait for balancer to come online
	TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
	@Override
	public boolean evaluate() throws Exception {
	return MASTER.isInitialized() &&
	((RSGroupBasedLoadBalancer) MASTER.getLoadBalancer()).isOnline();
	}
	});
	ADMIN.balancerSwitch(false, true);
	MasterCoprocessorHost host = MASTER.getMasterCoprocessorHost();
	OBSERVER = (CPMasterObserver) host.findCoprocessor(CPMasterObserver.class.getName());
	}

	public static void tearDownAfterClass() throws Exception {
	TEST_UTIL.shutdownMiniCluster();
	}

	public void setUpBeforeMethod() throws Exception {
	LOG.info(name.getMethodName());
	tableName = TableName.valueOf(TABLE_PREFIX + "_" + name.getMethodName().split("\\[")[0]);
	if (!INIT) {
	INIT = true;
	tearDownAfterMethod();
	}
	OBSERVER.resetFlags();
	}

	public void tearDownAfterMethod() throws Exception {
	deleteTableIfNecessary();
	deleteNamespaceIfNecessary();
	deleteGroups();

	for (ServerName sn : ADMIN.listDecommissionedRegionServers()) {
	ADMIN.recommissionRegionServer(sn, null);
	}
	assertTrue(ADMIN.listDecommissionedRegionServers().isEmpty());

	int missing = NUM_SLAVES_BASE - getNumServers();
	LOG.info("Restoring servers: " + missing);
	for (int i = 0; i < missing; i++) {
	((MiniHBaseCluster) CLUSTER).startRegionServer();
	}
	ADMIN.addRSGroup("master");
	ServerName masterServerName = ((MiniHBaseCluster) CLUSTER).getMaster().getServerName();
	try {
	ADMIN.moveServersToRSGroup(Sets.newHashSet(masterServerName.getAddress()), "master");
	} catch (Exception ex) {
	LOG.warn("Got this on setup, FYI", ex);
	}
	assertTrue(OBSERVER.preMoveServersCalled);
	TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
	@Override
	public boolean evaluate() throws Exception {
	LOG.info("Waiting for cleanup to finish " + ADMIN.listRSGroups());
	// Might be greater since moving servers back to default
	// is after starting a server

	return ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP).getServers().size() == NUM_SLAVES_BASE;
	}
	});
	}

	protected final RSGroupInfo addGroup(String groupName, int serverCount)
	throws IOException, InterruptedException {
	RSGroupInfo defaultInfo = ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP);
	ADMIN.addRSGroup(groupName);
	Set<Address> set = new HashSet<>();
	for (Address server : defaultInfo.getServers()) {
	if (set.size() == serverCount) {
	break;
	}
	set.add(server);
	}
	ADMIN.moveServersToRSGroup(set, groupName);
	RSGroupInfo result = ADMIN.getRSGroup(groupName);
	return result;
	}

	protected final void removeGroup(String groupName) throws IOException {
	Set<TableName> tables = new HashSet<>();
	for (TableDescriptor td : ADMIN.listTableDescriptors(true)) {
	RSGroupInfo groupInfo = ADMIN.getRSGroup(td.getTableName());
	if (groupInfo != null && groupInfo.getName().equals(groupName)) {
	tables.add(td.getTableName());
	}
	}
	ADMIN.setRSGroup(tables, RSGroupInfo.DEFAULT_GROUP);
	for (NamespaceDescriptor nd : ADMIN.listNamespaceDescriptors()) {
	if (groupName.equals(nd.getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP))) {
	nd.removeConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP);
	ADMIN.modifyNamespace(nd);
	}
	}
	RSGroupInfo groupInfo = ADMIN.getRSGroup(groupName);
	ADMIN.moveServersToRSGroup(groupInfo.getServers(), RSGroupInfo.DEFAULT_GROUP);
	ADMIN.removeRSGroup(groupName);
	}

	protected final void deleteTableIfNecessary() throws IOException {
	for (TableDescriptor desc : TEST_UTIL.getAdmin()
	.listTableDescriptors(Pattern.compile(TABLE_PREFIX + ".*"))) {
	TEST_UTIL.deleteTable(desc.getTableName());
	}
	}

	protected final void deleteNamespaceIfNecessary() throws IOException {
	for (NamespaceDescriptor desc : TEST_UTIL.getAdmin().listNamespaceDescriptors()) {
	if (desc.getName().startsWith(TABLE_PREFIX)) {
	ADMIN.deleteNamespace(desc.getName());
	}
	}
	}

	protected final void deleteGroups() throws IOException {
	for (RSGroupInfo groupInfo : ADMIN.listRSGroups()) {
	if (!groupInfo.getName().equals(RSGroupInfo.DEFAULT_GROUP)) {
	removeGroup(groupInfo.getName());
	}
	}
	}

	protected Map<TableName, List<String>> getTableRegionMap() throws IOException {
	Map<TableName, List<String>> map = Maps.newTreeMap();
	Map<TableName, Map<ServerName, List<String>>> tableServerRegionMap = getTableServerRegionMap();
	for (TableName tableName : tableServerRegionMap.keySet()) {
	if (!map.containsKey(tableName)) {
	map.put(tableName, new LinkedList<>());
	}
	for (List<String> subset : tableServerRegionMap.get(tableName).values()) {
	map.get(tableName).addAll(subset);
	}
	}
	return map;
	}

	protected Map<TableName, Map<ServerName, List<String>>> getTableServerRegionMap()
	throws IOException {
	Map<TableName, Map<ServerName, List<String>>> map = Maps.newTreeMap();
	Admin admin = TEST_UTIL.getAdmin();
	ClusterMetrics metrics =
	admin.getClusterMetrics(EnumSet.of(ClusterMetrics.Option.SERVERS_NAME));
	for (ServerName serverName : metrics.getServersName()) {
	for (RegionInfo region : admin.getRegions(serverName)) {
	TableName tableName = region.getTable();
	map.computeIfAbsent(tableName, k -> new TreeMap<>())
	.computeIfAbsent(serverName, k -> new ArrayList<>()).add(region.getRegionNameAsString());
	}
	}
	return map;
	}

	// return the real number of region servers, excluding the master embedded region server in 2.0+
	protected int getNumServers() throws IOException {
	ClusterMetrics status = ADMIN.getClusterMetrics(EnumSet.of(Option.MASTER, Option.LIVE_SERVERS));
	ServerName masterName = status.getMasterName();
	int count = 0;
	for (ServerName sn : status.getLiveServerMetrics().keySet()) {
	if (!sn.equals(masterName)) {
	count++;
	}
	}
	return count;
	}

	protected final String getGroupName(String baseName) {
	return GROUP_PREFIX + "_" + getNameWithoutIndex(baseName) + "_" +
	ThreadLocalRandom.current().nextInt(Integer.MAX_VALUE);
	}

	/**
	* The server name in group does not contain the start code, this method will find out the start
	* code and construct the ServerName object.
	*/
	protected final ServerName getServerName(Address addr) {
	return TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().stream()
	.map(t -> t.getRegionServer().getServerName()).filter(sn -> sn.getAddress().equals(addr))
	.findFirst().get();
	}

	protected final void toggleQuotaCheckAndRestartMiniCluster(boolean enable) throws Exception {
	TEST_UTIL.shutdownMiniCluster();
	TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, enable);
	TEST_UTIL.startMiniCluster(NUM_SLAVES_BASE - 1);
	TEST_UTIL.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART,
	NUM_SLAVES_BASE - 1);
	TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
	initialize();
	}

	public static class CPMasterObserver implements MasterCoprocessor, MasterObserver {
	boolean preBalanceRSGroupCalled = false;
	boolean postBalanceRSGroupCalled = false;
	boolean preMoveServersCalled = false;
	boolean postMoveServersCalled = false;
	boolean preMoveTablesCalled = false;
	boolean postMoveTablesCalled = false;
	boolean preAddRSGroupCalled = false;
	boolean postAddRSGroupCalled = false;
	boolean preRemoveRSGroupCalled = false;
	boolean postRemoveRSGroupCalled = false;
	boolean preRemoveServersCalled = false;
	boolean postRemoveServersCalled = false;
	boolean preMoveServersAndTables = false;
	boolean postMoveServersAndTables = false;
	boolean preGetRSGroupInfoCalled = false;
	boolean postGetRSGroupInfoCalled = false;
	boolean preGetRSGroupInfoOfTableCalled = false;
	boolean postGetRSGroupInfoOfTableCalled = false;
	boolean preListRSGroupsCalled = false;
	boolean postListRSGroupsCalled = false;
	boolean preGetRSGroupInfoOfServerCalled = false;
	boolean postGetRSGroupInfoOfServerCalled = false;
	boolean preSetRSGroupForTablesCalled = false;
	boolean postSetRSGroupForTablesCalled = false;
	boolean preListTablesInRSGroupCalled = false;
	boolean postListTablesInRSGroupCalled = false;
	boolean preGetConfiguredNamespacesAndTablesInRSGroupCalled = false;
	boolean postGetConfiguredNamespacesAndTablesInRSGroupCalled = false;
	boolean preRenameRSGroup = false;
	boolean postRenameRSGroup = false;

	public void resetFlags() {
	preBalanceRSGroupCalled = false;
	postBalanceRSGroupCalled = false;
	preMoveServersCalled = false;
	postMoveServersCalled = false;
	preMoveTablesCalled = false;
	postMoveTablesCalled = false;
	preAddRSGroupCalled = false;
	postAddRSGroupCalled = false;
	preRemoveRSGroupCalled = false;
	postRemoveRSGroupCalled = false;
	preRemoveServersCalled = false;
	postRemoveServersCalled = false;
	preMoveServersAndTables = false;
	postMoveServersAndTables = false;
	preGetRSGroupInfoCalled = false;
	postGetRSGroupInfoCalled = false;
	preGetRSGroupInfoOfTableCalled = false;
	postGetRSGroupInfoOfTableCalled = false;
	preListRSGroupsCalled = false;
	postListRSGroupsCalled = false;
	preGetRSGroupInfoOfServerCalled = false;
	postGetRSGroupInfoOfServerCalled = false;
	preSetRSGroupForTablesCalled = false;
	postSetRSGroupForTablesCalled = false;
	preListTablesInRSGroupCalled = false;
	postListTablesInRSGroupCalled = false;
	preGetConfiguredNamespacesAndTablesInRSGroupCalled = false;
	postGetConfiguredNamespacesAndTablesInRSGroupCalled = false;
	preRenameRSGroup = false;
	postRenameRSGroup = false;
	}

	@Override
	public Optional<MasterObserver> getMasterObserver() {
	return Optional.of(this);
	}

	@Override
	public void preMoveServersAndTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
	Set<Address> servers, Set<TableName> tables, String targetGroup) throws IOException {
	preMoveServersAndTables = true;
	}

	@Override
	public void postMoveServersAndTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
	Set<Address> servers, Set<TableName> tables, String targetGroup) throws IOException {
	postMoveServersAndTables = true;
	}

	@Override
	public void preRemoveServers(final ObserverContext<MasterCoprocessorEnvironment> ctx,
	Set<Address> servers) throws IOException {
	preRemoveServersCalled = true;
	}

	@Override
	public void postRemoveServers(final ObserverContext<MasterCoprocessorEnvironment> ctx,
	Set<Address> servers) throws IOException {
	postRemoveServersCalled = true;
	}

	@Override
	public void preRemoveRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
	String name) throws IOException {
	preRemoveRSGroupCalled = true;
	}

	@Override
	public void postRemoveRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
	String name) throws IOException {
	postRemoveRSGroupCalled = true;
	}

	@Override
	public void preAddRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx, String name)
	throws IOException {
	preAddRSGroupCalled = true;
	}

	@Override
	public void postAddRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx, String name)
	throws IOException {
	postAddRSGroupCalled = true;
	}

	@Override
	public void preMoveTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
	Set<TableName> tables, String targetGroup) throws IOException {
	preMoveTablesCalled = true;
	}

	@Override
	public void postMoveTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
	Set<TableName> tables, String targetGroup) throws IOException {
	postMoveTablesCalled = true;
	}

	@Override
	public void preMoveServers(final ObserverContext<MasterCoprocessorEnvironment> ctx,
	Set<Address> servers, String targetGroup) throws IOException {
	preMoveServersCalled = true;
	}

	@Override
	public void postMoveServers(final ObserverContext<MasterCoprocessorEnvironment> ctx,
	Set<Address> servers, String targetGroup) throws IOException {
	postMoveServersCalled = true;
	}

	@Override
	public void preBalanceRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
	String groupName) throws IOException {
	preBalanceRSGroupCalled = true;
	}

	@Override
	public void postBalanceRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
	String groupName, boolean balancerRan) throws IOException {
	postBalanceRSGroupCalled = true;
	}

	@Override
	public void preGetRSGroupInfo(final ObserverContext<MasterCoprocessorEnvironment> ctx,
	final String groupName) throws IOException {
	preGetRSGroupInfoCalled = true;
	}

	@Override
	public void postGetRSGroupInfo(final ObserverContext<MasterCoprocessorEnvironment> ctx,
	final String groupName) throws IOException {
	postGetRSGroupInfoCalled = true;
	}

	@Override
	public void preGetRSGroupInfoOfTable(final ObserverContext<MasterCoprocessorEnvironment> ctx,
	final TableName tableName) throws IOException {
	preGetRSGroupInfoOfTableCalled = true;
	}

	@Override
	public void postGetRSGroupInfoOfTable(final ObserverContext<MasterCoprocessorEnvironment> ctx,
	final TableName tableName) throws IOException {
	postGetRSGroupInfoOfTableCalled = true;
	}

	@Override
	public void preListRSGroups(final ObserverContext<MasterCoprocessorEnvironment> ctx)
	throws IOException {
	preListRSGroupsCalled = true;
	}

	@Override
	public void postListRSGroups(final ObserverContext<MasterCoprocessorEnvironment> ctx)
	throws IOException {
	postListRSGroupsCalled = true;
	}

	@Override
	public void preGetRSGroupInfoOfServer(final ObserverContext<MasterCoprocessorEnvironment> ctx,
	final Address server) throws IOException {
	preGetRSGroupInfoOfServerCalled = true;
	}

	@Override
	public void postGetRSGroupInfoOfServer(final ObserverContext<MasterCoprocessorEnvironment> ctx,
	final Address server) throws IOException {
	postGetRSGroupInfoOfServerCalled = true;
	}

	@Override
	public void preListTablesInRSGroup(ObserverContext<MasterCoprocessorEnvironment> ctx,
	String groupName) throws IOException {
	preListTablesInRSGroupCalled = true;
	}

	@Override
	public void postListTablesInRSGroup(ObserverContext<MasterCoprocessorEnvironment> ctx,
	String groupName) throws IOException {
	postListTablesInRSGroupCalled = true;
	}

	@Override
	public void preGetConfiguredNamespacesAndTablesInRSGroup(
	ObserverContext<MasterCoprocessorEnvironment> ctx, String groupName) throws IOException {
	preGetConfiguredNamespacesAndTablesInRSGroupCalled = true;
	}

	@Override
	public void postGetConfiguredNamespacesAndTablesInRSGroup(
	ObserverContext<MasterCoprocessorEnvironment> ctx, String groupName) throws IOException {
	postGetConfiguredNamespacesAndTablesInRSGroupCalled = true;
	}

	@Override
	public void preRenameRSGroup(ObserverContext<MasterCoprocessorEnvironment> ctx, String oldName,
	String newName) throws IOException {
	preRenameRSGroup = true;
	}

	@Override
	public void postRenameRSGroup(ObserverContext<MasterCoprocessorEnvironment> ctx, String oldName,
	String newName) throws IOException {
	postRenameRSGroup = true;
	}
	}
	}