hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hbase.master.balancer;

import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.stream.Collectors;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.master.LoadBalancer;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.RackManager;
import org.apache.hadoop.hbase.master.RegionPlan;
import org.apache.hadoop.hbase.master.ServerManager;
import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster;
import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.MoveRegionAction;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.net.DNSToSwitchMapping;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
import org.mockito.Mockito;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

@Category({MasterTests.class, MediumTests.class})
public class TestBaseLoadBalancer extends BalancerTestBase {

  private static LoadBalancer loadBalancer;
  private static final Log LOG = LogFactory.getLog(TestBaseLoadBalancer.class);
  private static final ServerName master = ServerName.valueOf("fake-master", 0, 1L);
  private static RackManager rackManager;
  private static final int NUM_SERVERS = 15;
  private static ServerName[] servers = new ServerName[NUM_SERVERS];

  int[][] regionsAndServersMocks = new int[][] {
      // { num regions, num servers }
      new int[] { 0, 0 }, new int[] { 0, 1 }, new int[] { 1, 1 }, new int[] { 2, 1 },
      new int[] { 10, 1 }, new int[] { 1, 2 }, new int[] { 2, 2 }, new int[] { 3, 2 },
      new int[] { 1, 3 }, new int[] { 2, 3 }, new int[] { 3, 3 }, new int[] { 25, 3 },
      new int[] { 2, 10 }, new int[] { 2, 100 }, new int[] { 12, 10 }, new int[] { 12, 100 }, };

  @Rule
  public TestName name = new TestName();

  @BeforeClass
  public static void beforeAllTests() throws Exception {
    Configuration conf = HBaseConfiguration.create();
    conf.setClass("hbase.util.ip.to.rack.determiner", MockMapping.class, DNSToSwitchMapping.class);
    loadBalancer = new MockBalancer();
    loadBalancer.setConf(conf);
    MasterServices st = Mockito.mock(MasterServices.class);
    Mockito.when(st.getServerName()).thenReturn(master);
    loadBalancer.setMasterServices(st);

    // Set up the rack topologies (5 machines per rack)
    rackManager = Mockito.mock(RackManager.class);
    for (int i = 0; i < NUM_SERVERS; i++) {
      servers[i] = ServerName.valueOf("foo"+i+":1234",-1);
      if (i < 5) {
        Mockito.when(rackManager.getRack(servers[i])).thenReturn("rack1");
      }
      if (i >= 5 && i < 10) {
        Mockito.when(rackManager.getRack(servers[i])).thenReturn("rack2");
      }
      if (i >= 10) {
        Mockito.when(rackManager.getRack(servers[i])).thenReturn("rack3");
      }
    }
  }

  public static class MockBalancer extends BaseLoadBalancer {
    @Override
    public List<RegionPlan> balanceCluster(Map<ServerName, List<HRegionInfo>> clusterState) {
      return null;
    }

    @Override
    public List<RegionPlan> balanceCluster(TableName tableName,
        Map<ServerName, List<HRegionInfo>> clusterState) throws HBaseIOException {
      return null;
    }
  }

  /**
   * All regions have an assignment.
   * @param regions
   * @param servers
   * @param assignments
   */
  private void assertImmediateAssignment(List<HRegionInfo> regions, List<ServerName> servers,
      Map<HRegionInfo, ServerName> assignments) {
    for (HRegionInfo region : regions) {
      assertTrue(assignments.containsKey(region));
    }
  }

  /**
   * Tests the bulk assignment used during cluster startup.
   *
   * Round-robin. Should yield a balanced cluster so same invariant as the load
   * balancer holds, all servers holding either floor(avg) or ceiling(avg).
   *
   * @throws Exception
   */
  @Test (timeout=180000)
  public void testBulkAssignment() throws Exception {
    List<ServerName> tmp = getListOfServerNames(randomServers(5, 0));
    List<HRegionInfo> hris = randomRegions(20);
    hris.add(HRegionInfo.FIRST_META_REGIONINFO);
    tmp.add(master);
    Map<ServerName, List<HRegionInfo>> plans = loadBalancer.roundRobinAssignment(hris, tmp);
    if (LoadBalancer.isTablesOnMaster(loadBalancer.getConf())) {
      assertTrue(plans.get(master).contains(HRegionInfo.FIRST_META_REGIONINFO));
      assertEquals(1, plans.get(master).size());
    }
    int totalRegion = 0;
    for (List<HRegionInfo> regions: plans.values()) {
      totalRegion += regions.size();
    }
    assertEquals(hris.size(), totalRegion);
    for (int[] mock : regionsAndServersMocks) {
      LOG.debug("testBulkAssignment with " + mock[0] + " regions and " + mock[1] + " servers");
      List<HRegionInfo> regions = randomRegions(mock[0]);
      List<ServerAndLoad> servers = randomServers(mock[1], 0);
      List<ServerName> list = getListOfServerNames(servers);
      Map<ServerName, List<HRegionInfo>> assignments =
          loadBalancer.roundRobinAssignment(regions, list);
      float average = (float) regions.size() / servers.size();
      int min = (int) Math.floor(average);
      int max = (int) Math.ceil(average);
      if (assignments != null && !assignments.isEmpty()) {
        for (List<HRegionInfo> regionList : assignments.values()) {
          assertTrue(regionList.size() == min || regionList.size() == max);
        }
      }
      returnRegions(regions);
      returnServers(list);
    }
  }

  /**
   * Test the cluster startup bulk assignment which attempts to retain
   * assignment info.
   * @throws Exception
   */
  @Test (timeout=180000)
  public void testRetainAssignment() throws Exception {
    // Test simple case where all same servers are there
    List<ServerAndLoad> servers = randomServers(10, 10);
    List<HRegionInfo> regions = randomRegions(100);
    Map<HRegionInfo, ServerName> existing = new TreeMap<>();
    for (int i = 0; i < regions.size(); i++) {
      ServerName sn = servers.get(i % servers.size()).getServerName();
      // The old server would have had same host and port, but different
      // start code!
      ServerName snWithOldStartCode =
          ServerName.valueOf(sn.getHostname(), sn.getPort(), sn.getStartcode() - 10);
      existing.put(regions.get(i), snWithOldStartCode);
    }
    List<ServerName> listOfServerNames = getListOfServerNames(servers);
    Map<ServerName, List<HRegionInfo>> assignment =
        loadBalancer.retainAssignment(existing, listOfServerNames);
    assertRetainedAssignment(existing, listOfServerNames, assignment);

    // Include two new servers that were not there before
    List<ServerAndLoad> servers2 = new ArrayList<>(servers);
    servers2.add(randomServer(10));
    servers2.add(randomServer(10));
    listOfServerNames = getListOfServerNames(servers2);
    assignment = loadBalancer.retainAssignment(existing, listOfServerNames);
    assertRetainedAssignment(existing, listOfServerNames, assignment);

    // Remove two of the servers that were previously there
    List<ServerAndLoad> servers3 = new ArrayList<>(servers);
    servers3.remove(0);
    servers3.remove(0);
    listOfServerNames = getListOfServerNames(servers3);
    assignment = loadBalancer.retainAssignment(existing, listOfServerNames);
    assertRetainedAssignment(existing, listOfServerNames, assignment);
  }

  @Test (timeout=30000)
  public void testRandomAssignment() throws Exception {
    for (int i = 1; i != 5; ++i) {
      LOG.info("run testRandomAssignment() with idle servers:" + i);
      testRandomAssignment(i);
    }
  }

  private void testRandomAssignment(int numberOfIdleServers) throws Exception {
    assert numberOfIdleServers > 0;
    List<ServerName> idleServers = new ArrayList<>(numberOfIdleServers);
    for (int i = 0; i != numberOfIdleServers; ++i) {
      idleServers.add(ServerName.valueOf("server-" + i, 1000, 1L));
    }
    List<ServerName> allServers = new ArrayList<>(idleServers.size() + 1);
    allServers.add(ServerName.valueOf("server-" + numberOfIdleServers, 1000, 1L));
    allServers.addAll(idleServers);
    LoadBalancer balancer = new MockBalancer() {
      @Override
      public boolean shouldBeOnMaster(HRegionInfo region) {
        return false;
      }
    };
    Configuration conf = HBaseConfiguration.create();
    conf.setClass("hbase.util.ip.to.rack.determiner", MockMapping.class, DNSToSwitchMapping.class);
    balancer.setConf(conf);
    ServerManager sm = Mockito.mock(ServerManager.class);
    Mockito.when(sm.getOnlineServersListWithPredicator(allServers, BaseLoadBalancer.IDLE_SERVER_PREDICATOR))
           .thenReturn(idleServers);
    MasterServices services = Mockito.mock(MasterServices.class);
    Mockito.when(services.getServerManager()).thenReturn(sm);
    balancer.setMasterServices(services);
    HRegionInfo hri1 = new HRegionInfo(
        TableName.valueOf(name.getMethodName()), "key1".getBytes(), "key2".getBytes(),
        false, 100);
    assertNull(balancer.randomAssignment(hri1, Collections.EMPTY_LIST));
    assertNull(balancer.randomAssignment(hri1, null));
    for (int i = 0; i != 3; ++i) {
      ServerName sn = balancer.randomAssignment(hri1, allServers);
      assertTrue("actual:" + sn + ", except:" + idleServers, idleServers.contains(sn));
    }
  }

  @Test (timeout=180000)
  public void testRegionAvailability() throws Exception {
    // Create a cluster with a few servers, assign them to specific racks
    // then assign some regions. The tests should check whether moving a
    // replica from one node to a specific other node or rack lowers the
    // availability of the region or not

    List<HRegionInfo> list0 = new ArrayList<>();
    List<HRegionInfo> list1 = new ArrayList<>();
    List<HRegionInfo> list2 = new ArrayList<>();
    // create a region (region1)
    HRegionInfo hri1 = new HRegionInfo(
        TableName.valueOf(name.getMethodName()), "key1".getBytes(), "key2".getBytes(),
        false, 100);
    // create a replica of the region (replica_of_region1)
    HRegionInfo hri2 = RegionReplicaUtil.getRegionInfoForReplica(hri1, 1);
    // create a second region (region2)
    HRegionInfo hri3 = new HRegionInfo(
        TableName.valueOf(name.getMethodName()), "key2".getBytes(), "key3".getBytes(),
        false, 101);
    list0.add(hri1); //only region1
    list1.add(hri2); //only replica_of_region1
    list2.add(hri3); //only region2
    Map<ServerName, List<HRegionInfo>> clusterState = new LinkedHashMap<>();
    clusterState.put(servers[0], list0); //servers[0] hosts region1
    clusterState.put(servers[1], list1); //servers[1] hosts replica_of_region1
    clusterState.put(servers[2], list2); //servers[2] hosts region2
    // create a cluster with the above clusterState. The way in which the
    // cluster is created (constructor code) would make sure the indices of
    // the servers are in the order in which it is inserted in the clusterState
    // map (linkedhashmap is important). A similar thing applies to the region lists
    Cluster cluster = new Cluster(clusterState, null, null, rackManager);
    // check whether a move of region1 from servers[0] to servers[1] would lower
    // the availability of region1
    assertTrue(cluster.wouldLowerAvailability(hri1, servers[1]));
    // check whether a move of region1 from servers[0] to servers[2] would lower
    // the availability of region1
    assertTrue(!cluster.wouldLowerAvailability(hri1, servers[2]));
    // check whether a move of replica_of_region1 from servers[0] to servers[2] would lower
    // the availability of replica_of_region1
    assertTrue(!cluster.wouldLowerAvailability(hri2, servers[2]));
    // check whether a move of region2 from servers[0] to servers[1] would lower
    // the availability of region2
    assertTrue(!cluster.wouldLowerAvailability(hri3, servers[1]));

    // now lets have servers[1] host replica_of_region2
    list1.add(RegionReplicaUtil.getRegionInfoForReplica(hri3, 1));
    // create a new clusterState with the above change
    cluster = new Cluster(clusterState, null, null, rackManager);
    // now check whether a move of a replica from servers[0] to servers[1] would lower
    // the availability of region2
    assertTrue(cluster.wouldLowerAvailability(hri3, servers[1]));

    // start over again
    clusterState.clear();
    clusterState.put(servers[0], list0); //servers[0], rack1 hosts region1
    clusterState.put(servers[5], list1); //servers[5], rack2 hosts replica_of_region1 and replica_of_region2
    clusterState.put(servers[6], list2); //servers[6], rack2 hosts region2
    clusterState.put(servers[10], new ArrayList<>()); //servers[10], rack3 hosts no region
    // create a cluster with the above clusterState
    cluster = new Cluster(clusterState, null, null, rackManager);
    // check whether a move of region1 from servers[0],rack1 to servers[6],rack2 would
    // lower the availability

    assertTrue(cluster.wouldLowerAvailability(hri1, servers[0]));

    // now create a cluster without the rack manager
    cluster = new Cluster(clusterState, null, null, null);
    // now repeat check whether a move of region1 from servers[0] to servers[6] would
    // lower the availability
    assertTrue(!cluster.wouldLowerAvailability(hri1, servers[6]));
  }

  @Test (timeout=180000)
  public void testRegionAvailabilityWithRegionMoves() throws Exception {
    List<HRegionInfo> list0 = new ArrayList<>();
    List<HRegionInfo> list1 = new ArrayList<>();
    List<HRegionInfo> list2 = new ArrayList<>();
    // create a region (region1)
    HRegionInfo hri1 = new HRegionInfo(
        TableName.valueOf(name.getMethodName()), "key1".getBytes(), "key2".getBytes(),
        false, 100);
    // create a replica of the region (replica_of_region1)
    HRegionInfo hri2 = RegionReplicaUtil.getRegionInfoForReplica(hri1, 1);
    // create a second region (region2)
    HRegionInfo hri3 = new HRegionInfo(
        TableName.valueOf(name.getMethodName()), "key2".getBytes(), "key3".getBytes(),
        false, 101);
    list0.add(hri1); //only region1
    list1.add(hri2); //only replica_of_region1
    list2.add(hri3); //only region2
    Map<ServerName, List<HRegionInfo>> clusterState = new LinkedHashMap<>();
    clusterState.put(servers[0], list0); //servers[0] hosts region1
    clusterState.put(servers[1], list1); //servers[1] hosts replica_of_region1
    clusterState.put(servers[2], list2); //servers[2] hosts region2
    // create a cluster with the above clusterState. The way in which the
    // cluster is created (constructor code) would make sure the indices of
    // the servers are in the order in which it is inserted in the clusterState
    // map (linkedhashmap is important).
    Cluster cluster = new Cluster(clusterState, null, null, rackManager);
    // check whether moving region1 from servers[1] to servers[2] would lower availability
    assertTrue(!cluster.wouldLowerAvailability(hri1, servers[2]));

    // now move region1 from servers[0] to servers[2]
    cluster.doAction(new MoveRegionAction(0, 0, 2));
    // check that the numMaxRegionsPerTable for "table" has increased to 2
    assertEquals(2, cluster.numMaxRegionsPerTable[0]);
    // now repeat check whether moving region1 from servers[1] to servers[2]
    // would lower availability
    assertTrue(cluster.wouldLowerAvailability(hri1, servers[2]));

    // start over again
    clusterState.clear();
    List<HRegionInfo> list3 = new ArrayList<>();
    HRegionInfo hri4 = RegionReplicaUtil.getRegionInfoForReplica(hri3, 1);
    list3.add(hri4);
    clusterState.put(servers[0], list0); //servers[0], rack1 hosts region1
    clusterState.put(servers[5], list1); //servers[5], rack2 hosts replica_of_region1
    clusterState.put(servers[6], list2); //servers[6], rack2 hosts region2
    clusterState.put(servers[12], list3); //servers[12], rack3 hosts replica_of_region2
    // create a cluster with the above clusterState
    cluster = new Cluster(clusterState, null, null, rackManager);
    // check whether a move of replica_of_region2 from servers[12],rack3 to servers[0],rack1 would
    // lower the availability
    assertTrue(!cluster.wouldLowerAvailability(hri4, servers[0]));
    // now move region2 from servers[6],rack2 to servers[0],rack1
    cluster.doAction(new MoveRegionAction(2, 2, 0));
    // now repeat check if replica_of_region2 from servers[12],rack3 to servers[0],rack1 would
    // lower the availability
    assertTrue(cluster.wouldLowerAvailability(hri3, servers[0]));
  }

  private List<ServerName> getListOfServerNames(final List<ServerAndLoad> sals) {
    return sals.stream().map(ServerAndLoad::getServerName).collect(Collectors.toList());
  }

  /**
   * Asserts a valid retained assignment plan.
   * <p>
   * Must meet the following conditions:
   * <ul>
   * <li>Every input region has an assignment, and to an online server
   * <li>If a region had an existing assignment to a server with the same
   * address a a currently online server, it will be assigned to it
   * </ul>
   * @param existing
   * @param servers
   * @param assignment
   */
  private void assertRetainedAssignment(Map<HRegionInfo, ServerName> existing,
      List<ServerName> servers, Map<ServerName, List<HRegionInfo>> assignment) {
    // Verify condition 1, every region assigned, and to online server
    Set<ServerName> onlineServerSet = new TreeSet<>(servers);
    Set<HRegionInfo> assignedRegions = new TreeSet<>();
    for (Map.Entry<ServerName, List<HRegionInfo>> a : assignment.entrySet()) {
      assertTrue("Region assigned to server that was not listed as online",
        onlineServerSet.contains(a.getKey()));
      for (HRegionInfo r : a.getValue())
        assignedRegions.add(r);
    }
    assertEquals(existing.size(), assignedRegions.size());

    // Verify condition 2, if server had existing assignment, must have same
    Set<String> onlineHostNames = new TreeSet<>();
    for (ServerName s : servers) {
      onlineHostNames.add(s.getHostname());
    }

    for (Map.Entry<ServerName, List<HRegionInfo>> a : assignment.entrySet()) {
      ServerName assignedTo = a.getKey();
      for (HRegionInfo r : a.getValue()) {
        ServerName address = existing.get(r);
        if (address != null && onlineHostNames.contains(address.getHostname())) {
          // this region was prevously assigned somewhere, and that
          // host is still around, then it should be re-assigned on the
          // same host
          assertEquals(address.getHostname(), assignedTo.getHostname());
        }
      }
    }
  }

  @Test (timeout=180000)
  public void testClusterServersWithSameHostPort() {
    // tests whether the BaseLoadBalancer.Cluster can be constructed with servers
    // sharing same host and port
    List<ServerName> servers = getListOfServerNames(randomServers(10, 10));
    List<HRegionInfo> regions = randomRegions(101);
    Map<ServerName, List<HRegionInfo>> clusterState = new HashMap<>();

    assignRegions(regions, servers, clusterState);

    // construct another list of servers, but sharing same hosts and ports
    List<ServerName> oldServers = new ArrayList<>(servers.size());
    for (ServerName sn : servers) {
      // The old server would have had same host and port, but different start code!
      oldServers.add(ServerName.valueOf(sn.getHostname(), sn.getPort(), sn.getStartcode() - 10));
    }

    regions = randomRegions(9); // some more regions
    assignRegions(regions, oldServers, clusterState);

    // should not throw exception:
    BaseLoadBalancer.Cluster cluster = new Cluster(clusterState, null, null, null);
    assertEquals(101 + 9, cluster.numRegions);
    assertEquals(10, cluster.numServers); // only 10 servers because they share the same host + port
  }

  private void assignRegions(List<HRegionInfo> regions, List<ServerName> servers,
      Map<ServerName, List<HRegionInfo>> clusterState) {
    for (int i = 0; i < regions.size(); i++) {
      ServerName sn = servers.get(i % servers.size());
      List<HRegionInfo> regionsOfServer = clusterState.get(sn);
      if (regionsOfServer == null) {
        regionsOfServer = new ArrayList<>(10);
        clusterState.put(sn, regionsOfServer);
      }

      regionsOfServer.add(regions.get(i));
    }
  }

  @Test (timeout=180000)
  public void testClusterRegionLocations() {
    // tests whether region locations are handled correctly in Cluster
    List<ServerName> servers = getListOfServerNames(randomServers(10, 10));
    List<HRegionInfo> regions = randomRegions(101);
    Map<ServerName, List<HRegionInfo>> clusterState = new HashMap<>();

    assignRegions(regions, servers, clusterState);

    // mock block locality for some regions
    RegionLocationFinder locationFinder = mock(RegionLocationFinder.class);
    // block locality: region:0   => {server:0}
    //                 region:1   => {server:0, server:1}
    //                 region:42 => {server:4, server:9, server:5}
    when(locationFinder.getTopBlockLocations(regions.get(0))).thenReturn(
        Lists.newArrayList(servers.get(0)));
    when(locationFinder.getTopBlockLocations(regions.get(1))).thenReturn(
        Lists.newArrayList(servers.get(0), servers.get(1)));
    when(locationFinder.getTopBlockLocations(regions.get(42))).thenReturn(
        Lists.newArrayList(servers.get(4), servers.get(9), servers.get(5)));
    when(locationFinder.getTopBlockLocations(regions.get(43))).thenReturn(
        Lists.newArrayList(ServerName.valueOf("foo", 0, 0))); // this server does not exists in clusterStatus

    BaseLoadBalancer.Cluster cluster = new Cluster(clusterState, null, locationFinder, null);

    int r0 = ArrayUtils.indexOf(cluster.regions, regions.get(0)); // this is ok, it is just a test
    int r1 = ArrayUtils.indexOf(cluster.regions, regions.get(1));
    int r10 = ArrayUtils.indexOf(cluster.regions, regions.get(10));
    int r42 = ArrayUtils.indexOf(cluster.regions, regions.get(42));
    int r43 = ArrayUtils.indexOf(cluster.regions, regions.get(43));

    int s0 = cluster.serversToIndex.get(servers.get(0).getHostAndPort());
    int s1 = cluster.serversToIndex.get(servers.get(1).getHostAndPort());
    int s4 = cluster.serversToIndex.get(servers.get(4).getHostAndPort());
    int s5 = cluster.serversToIndex.get(servers.get(5).getHostAndPort());
    int s9 = cluster.serversToIndex.get(servers.get(9).getHostAndPort());

    // region 0 locations
    assertEquals(1, cluster.regionLocations[r0].length);
    assertEquals(s0, cluster.regionLocations[r0][0]);

    // region 1 locations
    assertEquals(2, cluster.regionLocations[r1].length);
    assertEquals(s0, cluster.regionLocations[r1][0]);
    assertEquals(s1, cluster.regionLocations[r1][1]);

    // region 10 locations
    assertEquals(0, cluster.regionLocations[r10].length);

    // region 42 locations
    assertEquals(3, cluster.regionLocations[r42].length);
    assertEquals(s4, cluster.regionLocations[r42][0]);
    assertEquals(s9, cluster.regionLocations[r42][1]);
    assertEquals(s5, cluster.regionLocations[r42][2]);

    // region 43 locations
    assertEquals(1, cluster.regionLocations[r43].length);
    assertEquals(-1, cluster.regionLocations[r43][0]);
  }
}