hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java - hbase - Git at Google

 /**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
  * regarding copyright ownership.  The ASF licenses this file
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package org.apache.hadoop.hbase.tool;

 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.ArgumentMatchers.argThat;
 import static org.mockito.Matchers.anyLong;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Matchers.isA;
 import static org.mockito.Mockito.atLeastOnce;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;

 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
 import org.apache.hadoop.hbase.HBaseClassTestRule;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.RegionInfo;
 import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.util.ToolRunner;
 import org.apache.log4j.Appender;
 import org.apache.log4j.LogManager;
 import org.apache.log4j.spi.LoggingEvent;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.ClassRule;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.junit.rules.TestName;
 import org.junit.runner.RunWith;
 import org.mockito.ArgumentMatcher;
 import org.mockito.Mock;
 import org.mockito.runners.MockitoJUnitRunner;

 import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;

 @RunWith(MockitoJUnitRunner.class)
 @Category({MediumTests.class})
 public class TestCanaryTool {

   @ClassRule
   public static final HBaseClassTestRule CLASS_RULE =
       HBaseClassTestRule.forClass(TestCanaryTool.class);

   private HBaseTestingUtility testingUtility;
   private static final byte[] FAMILY = Bytes.toBytes("f");
   private static final byte[] COLUMN = Bytes.toBytes("col");

   @Rule
   public TestName name = new TestName();

   @Before
   public void setUp() throws Exception {
     testingUtility = new HBaseTestingUtility();
     testingUtility.startMiniCluster();
     LogManager.getRootLogger().addAppender(mockAppender);
   }

   @After
   public void tearDown() throws Exception {
     testingUtility.shutdownMiniCluster();
     LogManager.getRootLogger().removeAppender(mockAppender);
   }

   @Mock
   Appender mockAppender;

   @Test
   public void testBasicZookeeperCanaryWorks() throws Exception {
     final String[] args = { "-t", "10000", "-zookeeper" };
     testZookeeperCanaryWithArgs(args);
   }

   @Test
   public void testZookeeperCanaryPermittedFailuresArgumentWorks() throws Exception {
     final String[] args = { "-t", "10000", "-zookeeper", "-treatFailureAsError", "-permittedZookeeperFailures", "1" };
     testZookeeperCanaryWithArgs(args);
   }

   @Test
   public void testBasicCanaryWorks() throws Exception {
     final TableName tableName = TableName.valueOf(name.getMethodName());
     Table table = testingUtility.createTable(tableName, new byte[][] { FAMILY });
     // insert some test rows
     for (int i=0; i<1000; i++) {
       byte[] iBytes = Bytes.toBytes(i);
       Put p = new Put(iBytes);
       p.addColumn(FAMILY, COLUMN, iBytes);
       table.put(p);
     }
     ExecutorService executor = new ScheduledThreadPoolExecutor(1);
     CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
     CanaryTool canary = new CanaryTool(executor, sink);
     String[] args = { "-writeSniffing", "-t", "10000", tableName.getNameAsString() };
     assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
     assertEquals("verify no read error count", 0, canary.getReadFailures().size());
     assertEquals("verify no write error count", 0, canary.getWriteFailures().size());
     verify(sink, atLeastOnce()).publishReadTiming(isA(ServerName.class), isA(RegionInfo.class),
       isA(ColumnFamilyDescriptor.class), anyLong());
   }

   @Test
   public void testCanaryRegionTaskResult() throws Exception {
     TableName tableName = TableName.valueOf("testCanaryRegionTaskResult");
     Table table = testingUtility.createTable(tableName, new byte[][] { FAMILY });
     // insert some test rows
     for (int i=0; i<1000; i++) {
       byte[] iBytes = Bytes.toBytes(i);
       Put p = new Put(iBytes);
       p.addColumn(FAMILY, COLUMN, iBytes);
       table.put(p);
     }
     ExecutorService executor = new ScheduledThreadPoolExecutor(1);
     CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
     CanaryTool canary = new CanaryTool(executor, sink);
     String[] args = { "-writeSniffing", "-t", "10000", "testCanaryRegionTaskResult" };
     assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));

     assertTrue("canary should expect to scan at least 1 region",
         sink.getTotalExpectedRegions() > 0);
     assertTrue("there should be no read failures", sink.getReadFailureCount() == 0);
     assertTrue("there should be no write failures", sink.getWriteFailureCount() == 0);
     assertTrue("verify read success count > 0", sink.getReadSuccessCount() > 0);
     assertTrue("verify write success count > 0", sink.getWriteSuccessCount() > 0);
     verify(sink, atLeastOnce()).publishReadTiming(isA(ServerName.class), isA(RegionInfo.class),
         isA(ColumnFamilyDescriptor.class), anyLong());
     verify(sink, atLeastOnce()).publishWriteTiming(isA(ServerName.class), isA(RegionInfo.class),
         isA(ColumnFamilyDescriptor.class), anyLong());

     assertEquals("canary region success count should equal total expected regions",
         sink.getReadSuccessCount() + sink.getWriteSuccessCount(), sink.getTotalExpectedRegions());
     Map<String, List<CanaryTool.RegionTaskResult>> regionMap = sink.getRegionMap();
     assertFalse("verify region map has size > 0", regionMap.isEmpty());

     for (String regionName : regionMap.keySet()) {
       for (CanaryTool.RegionTaskResult res: regionMap.get(regionName)) {
         assertNotNull("verify getRegionNameAsString()", regionName);
         assertNotNull("verify getRegionInfo()", res.getRegionInfo());
         assertNotNull("verify getTableName()", res.getTableName());
         assertNotNull("verify getTableNameAsString()", res.getTableNameAsString());
         assertNotNull("verify getServerName()", res.getServerName());
         assertNotNull("verify getServerNameAsString()", res.getServerNameAsString());
         assertNotNull("verify getColumnFamily()", res.getColumnFamily());
         assertNotNull("verify getColumnFamilyNameAsString()", res.getColumnFamilyNameAsString());

         if (regionName.contains(CanaryTool.DEFAULT_WRITE_TABLE_NAME.getNameAsString())) {
           assertTrue("write to region " + regionName + " succeeded", res.isWriteSuccess());
           assertTrue("write took some time", res.getWriteLatency() > -1);
         } else {
           assertTrue("read from region " + regionName + " succeeded", res.isReadSuccess());
           assertTrue("read took some time", res.getReadLatency() > -1);
         }
       }
     }
   }

   @Test
   public void testReadTableTimeouts() throws Exception {
     final TableName [] tableNames = new TableName[2];
     tableNames[0] = TableName.valueOf(name.getMethodName() + "1");
     tableNames[1] = TableName.valueOf(name.getMethodName() + "2");
     // Create 2 test tables.
     for (int j = 0; j<2; j++) {
       Table table = testingUtility.createTable(tableNames[j], new byte[][] { FAMILY });
       // insert some test rows
       for (int i=0; i<1000; i++) {
         byte[] iBytes = Bytes.toBytes(i + j);
         Put p = new Put(iBytes);
         p.addColumn(FAMILY, COLUMN, iBytes);
         table.put(p);
       }
     }
     ExecutorService executor = new ScheduledThreadPoolExecutor(1);
     CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
     CanaryTool canary = new CanaryTool(executor, sink);
     String configuredTimeoutStr = tableNames[0].getNameAsString() + "=" + Long.MAX_VALUE + "," +
       tableNames[1].getNameAsString() + "=0";
     String[] args = {"-readTableTimeouts", configuredTimeoutStr, name.getMethodName() + "1",
       name.getMethodName() + "2"};
     assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
     verify(sink, times(tableNames.length)).initializeAndGetReadLatencyForTable(isA(String.class));
     for (int i=0; i<2; i++) {
       assertNotEquals("verify non-null read latency", null, sink.getReadLatencyMap().get(tableNames[i].getNameAsString()));
       assertNotEquals("verify non-zero read latency", 0L, sink.getReadLatencyMap().get(tableNames[i].getNameAsString()));
     }
     // One table's timeout is set for 0 ms and thus, should lead to an error.
     verify(mockAppender, times(1)).doAppend(argThat(new ArgumentMatcher<LoggingEvent>() {
       @Override
       public boolean matches(LoggingEvent argument) {
         return ((LoggingEvent) argument).getRenderedMessage().contains("exceeded the configured read timeout.");
       }
     }));
     verify(mockAppender, times(2)).doAppend(argThat(new ArgumentMatcher<LoggingEvent>() {
       @Override
       public boolean matches(LoggingEvent argument) {
         return argument.getRenderedMessage().contains("Configured read timeout");
       }
     }));
   }

   @Test
   public void testWriteTableTimeout() throws Exception {
     ExecutorService executor = new ScheduledThreadPoolExecutor(1);
     CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
     CanaryTool canary = new CanaryTool(executor, sink);
     String[] args = { "-writeSniffing", "-writeTableTimeout", String.valueOf(Long.MAX_VALUE)};
     assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
     assertNotEquals("verify non-null write latency", null, sink.getWriteLatency());
     assertNotEquals("verify non-zero write latency", 0L, sink.getWriteLatency());
     verify(mockAppender, times(1)).doAppend(argThat(
         new ArgumentMatcher<LoggingEvent>() {
           @Override
           public boolean matches(LoggingEvent argument) {
             return argument.getRenderedMessage().contains("Configured write timeout");
           }
         }));
   }

   //no table created, so there should be no regions
   @Test
   public void testRegionserverNoRegions() throws Exception {
     runRegionserverCanary();
     verify(mockAppender).doAppend(argThat(new ArgumentMatcher<LoggingEvent>() {
       @Override
       public boolean matches(LoggingEvent argument) {
         return argument.getRenderedMessage().contains("Regionserver not serving any regions");
       }
     }));
   }

   //by creating a table, there shouldn't be any region servers not serving any regions
   @Test
   public void testRegionserverWithRegions() throws Exception {
     final TableName tableName = TableName.valueOf(name.getMethodName());
     testingUtility.createTable(tableName, new byte[][] { FAMILY });
     runRegionserverCanary();
     verify(mockAppender, never()).doAppend(argThat(new ArgumentMatcher<LoggingEvent>() {
       @Override
       public boolean matches(LoggingEvent argument) {
         return argument.getRenderedMessage().contains("Regionserver not serving any regions");
       }
     }));
   }

   @Test
   public void testRawScanConfig() throws Exception {
     final TableName tableName = TableName.valueOf(name.getMethodName());
     Table table = testingUtility.createTable(tableName, new byte[][] { FAMILY });
     // insert some test rows
     for (int i=0; i<1000; i++) {
       byte[] iBytes = Bytes.toBytes(i);
       Put p = new Put(iBytes);
       p.addColumn(FAMILY, COLUMN, iBytes);
       table.put(p);
     }
     ExecutorService executor = new ScheduledThreadPoolExecutor(1);
     CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
     CanaryTool canary = new CanaryTool(executor, sink);
     String[] args = { "-t", "10000", name.getMethodName() };
     org.apache.hadoop.conf.Configuration conf =
       new org.apache.hadoop.conf.Configuration(testingUtility.getConfiguration());
     conf.setBoolean(HConstants.HBASE_CANARY_READ_RAW_SCAN_KEY, true);
     assertEquals(0, ToolRunner.run(conf, canary, args));
     verify(sink, atLeastOnce())
         .publishReadTiming(isA(ServerName.class), isA(RegionInfo.class),
         isA(ColumnFamilyDescriptor.class), anyLong());
     assertEquals("verify no read error count", 0, canary.getReadFailures().size());
   }

   private void runRegionserverCanary() throws Exception {
     ExecutorService executor = new ScheduledThreadPoolExecutor(1);
     CanaryTool canary = new CanaryTool(executor, new CanaryTool.RegionServerStdOutSink());
     String[] args = { "-t", "10000", "-regionserver"};
     assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
     assertEquals("verify no read error count", 0, canary.getReadFailures().size());
   }

   private void testZookeeperCanaryWithArgs(String[] args) throws Exception {
     Integer port =
       Iterables.getOnlyElement(testingUtility.getZkCluster().getClientPortList(), null);
     testingUtility.getConfiguration().set(HConstants.ZOOKEEPER_QUORUM, "localhost:" + port);
     ExecutorService executor = new ScheduledThreadPoolExecutor(2);
     CanaryTool.ZookeeperStdOutSink sink = spy(new CanaryTool.ZookeeperStdOutSink());
     CanaryTool canary = new CanaryTool(executor, sink);
     assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));

     String baseZnode = testingUtility.getConfiguration()
       .get(HConstants.ZOOKEEPER_ZNODE_PARENT, HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
     verify(sink, atLeastOnce())
       .publishReadTiming(eq(baseZnode), eq("localhost:" + port), anyLong());
   }
 }
	/**
	* Licensed to the Apache Software Foundation (ASF) under one
	* or more contributor license agreements. See the NOTICE file
	* distributed with this work for additional information
	* regarding copyright ownership. The ASF licenses this file
	* to you under the Apache License, Version 2.0 (the
	* "License"); you may not use this file except in compliance
	* with the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/
	package org.apache.hadoop.hbase.tool;

	import static org.junit.Assert.assertEquals;
	import static org.junit.Assert.assertFalse;
	import static org.junit.Assert.assertNotEquals;
	import static org.junit.Assert.assertNotNull;
	import static org.junit.Assert.assertTrue;
	import static org.mockito.ArgumentMatchers.argThat;
	import static org.mockito.Matchers.anyLong;
	import static org.mockito.Matchers.eq;
	import static org.mockito.Matchers.isA;
	import static org.mockito.Mockito.atLeastOnce;
	import static org.mockito.Mockito.never;
	import static org.mockito.Mockito.spy;
	import static org.mockito.Mockito.times;
	import static org.mockito.Mockito.verify;

	import java.util.List;
	import java.util.Map;
	import java.util.concurrent.ExecutorService;
	import java.util.concurrent.ScheduledThreadPoolExecutor;
	import org.apache.hadoop.hbase.HBaseClassTestRule;
	import org.apache.hadoop.hbase.HBaseTestingUtility;
	import org.apache.hadoop.hbase.HConstants;
	import org.apache.hadoop.hbase.ServerName;
	import org.apache.hadoop.hbase.TableName;
	import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
	import org.apache.hadoop.hbase.client.Put;
	import org.apache.hadoop.hbase.client.RegionInfo;
	import org.apache.hadoop.hbase.client.Table;
	import org.apache.hadoop.hbase.testclassification.MediumTests;
	import org.apache.hadoop.hbase.util.Bytes;
	import org.apache.hadoop.util.ToolRunner;
	import org.apache.log4j.Appender;
	import org.apache.log4j.LogManager;
	import org.apache.log4j.spi.LoggingEvent;
	import org.junit.After;
	import org.junit.Before;
	import org.junit.ClassRule;
	import org.junit.Rule;
	import org.junit.Test;
	import org.junit.experimental.categories.Category;
	import org.junit.rules.TestName;
	import org.junit.runner.RunWith;
	import org.mockito.ArgumentMatcher;
	import org.mockito.Mock;
	import org.mockito.runners.MockitoJUnitRunner;

	import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;

	@RunWith(MockitoJUnitRunner.class)
	@Category({MediumTests.class})
	public class TestCanaryTool {

	@ClassRule
	public static final HBaseClassTestRule CLASS_RULE =
	HBaseClassTestRule.forClass(TestCanaryTool.class);

	private HBaseTestingUtility testingUtility;
	private static final byte[] FAMILY = Bytes.toBytes("f");
	private static final byte[] COLUMN = Bytes.toBytes("col");

	@Rule
	public TestName name = new TestName();

	@Before
	public void setUp() throws Exception {
	testingUtility = new HBaseTestingUtility();
	testingUtility.startMiniCluster();
	LogManager.getRootLogger().addAppender(mockAppender);
	}

	@After
	public void tearDown() throws Exception {
	testingUtility.shutdownMiniCluster();
	LogManager.getRootLogger().removeAppender(mockAppender);
	}

	@Mock
	Appender mockAppender;

	@Test
	public void testBasicZookeeperCanaryWorks() throws Exception {
	final String[] args = { "-t", "10000", "-zookeeper" };
	testZookeeperCanaryWithArgs(args);
	}

	@Test
	public void testZookeeperCanaryPermittedFailuresArgumentWorks() throws Exception {
	final String[] args = { "-t", "10000", "-zookeeper", "-treatFailureAsError", "-permittedZookeeperFailures", "1" };
	testZookeeperCanaryWithArgs(args);
	}

	@Test
	public void testBasicCanaryWorks() throws Exception {
	final TableName tableName = TableName.valueOf(name.getMethodName());
	Table table = testingUtility.createTable(tableName, new byte[][] { FAMILY });
	// insert some test rows
	for (int i=0; i<1000; i++) {
	byte[] iBytes = Bytes.toBytes(i);
	Put p = new Put(iBytes);
	p.addColumn(FAMILY, COLUMN, iBytes);
	table.put(p);
	}
	ExecutorService executor = new ScheduledThreadPoolExecutor(1);
	CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
	CanaryTool canary = new CanaryTool(executor, sink);
	String[] args = { "-writeSniffing", "-t", "10000", tableName.getNameAsString() };
	assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
	assertEquals("verify no read error count", 0, canary.getReadFailures().size());
	assertEquals("verify no write error count", 0, canary.getWriteFailures().size());
	verify(sink, atLeastOnce()).publishReadTiming(isA(ServerName.class), isA(RegionInfo.class),
	isA(ColumnFamilyDescriptor.class), anyLong());
	}

	@Test
	public void testCanaryRegionTaskResult() throws Exception {
	TableName tableName = TableName.valueOf("testCanaryRegionTaskResult");
	Table table = testingUtility.createTable(tableName, new byte[][] { FAMILY });
	// insert some test rows
	for (int i=0; i<1000; i++) {
	byte[] iBytes = Bytes.toBytes(i);
	Put p = new Put(iBytes);
	p.addColumn(FAMILY, COLUMN, iBytes);
	table.put(p);
	}
	ExecutorService executor = new ScheduledThreadPoolExecutor(1);
	CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
	CanaryTool canary = new CanaryTool(executor, sink);
	String[] args = { "-writeSniffing", "-t", "10000", "testCanaryRegionTaskResult" };
	assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));

	assertTrue("canary should expect to scan at least 1 region",
	sink.getTotalExpectedRegions() > 0);
	assertTrue("there should be no read failures", sink.getReadFailureCount() == 0);
	assertTrue("there should be no write failures", sink.getWriteFailureCount() == 0);
	assertTrue("verify read success count > 0", sink.getReadSuccessCount() > 0);
	assertTrue("verify write success count > 0", sink.getWriteSuccessCount() > 0);
	verify(sink, atLeastOnce()).publishReadTiming(isA(ServerName.class), isA(RegionInfo.class),
	isA(ColumnFamilyDescriptor.class), anyLong());
	verify(sink, atLeastOnce()).publishWriteTiming(isA(ServerName.class), isA(RegionInfo.class),
	isA(ColumnFamilyDescriptor.class), anyLong());

	assertEquals("canary region success count should equal total expected regions",
	sink.getReadSuccessCount() + sink.getWriteSuccessCount(), sink.getTotalExpectedRegions());
	Map<String, List<CanaryTool.RegionTaskResult>> regionMap = sink.getRegionMap();
	assertFalse("verify region map has size > 0", regionMap.isEmpty());

	for (String regionName : regionMap.keySet()) {
	for (CanaryTool.RegionTaskResult res: regionMap.get(regionName)) {
	assertNotNull("verify getRegionNameAsString()", regionName);
	assertNotNull("verify getRegionInfo()", res.getRegionInfo());
	assertNotNull("verify getTableName()", res.getTableName());
	assertNotNull("verify getTableNameAsString()", res.getTableNameAsString());
	assertNotNull("verify getServerName()", res.getServerName());
	assertNotNull("verify getServerNameAsString()", res.getServerNameAsString());
	assertNotNull("verify getColumnFamily()", res.getColumnFamily());
	assertNotNull("verify getColumnFamilyNameAsString()", res.getColumnFamilyNameAsString());

	if (regionName.contains(CanaryTool.DEFAULT_WRITE_TABLE_NAME.getNameAsString())) {
	assertTrue("write to region " + regionName + " succeeded", res.isWriteSuccess());
	assertTrue("write took some time", res.getWriteLatency() > -1);
	} else {
	assertTrue("read from region " + regionName + " succeeded", res.isReadSuccess());
	assertTrue("read took some time", res.getReadLatency() > -1);
	}
	}
	}
	}

	@Test
	public void testReadTableTimeouts() throws Exception {
	final TableName [] tableNames = new TableName[2];
	tableNames[0] = TableName.valueOf(name.getMethodName() + "1");
	tableNames[1] = TableName.valueOf(name.getMethodName() + "2");
	// Create 2 test tables.
	for (int j = 0; j<2; j++) {
	Table table = testingUtility.createTable(tableNames[j], new byte[][] { FAMILY });
	// insert some test rows
	for (int i=0; i<1000; i++) {
	byte[] iBytes = Bytes.toBytes(i + j);
	Put p = new Put(iBytes);
	p.addColumn(FAMILY, COLUMN, iBytes);
	table.put(p);
	}
	}
	ExecutorService executor = new ScheduledThreadPoolExecutor(1);
	CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
	CanaryTool canary = new CanaryTool(executor, sink);
	String configuredTimeoutStr = tableNames[0].getNameAsString() + "=" + Long.MAX_VALUE + "," +
	tableNames[1].getNameAsString() + "=0";
	String[] args = {"-readTableTimeouts", configuredTimeoutStr, name.getMethodName() + "1",
	name.getMethodName() + "2"};
	assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
	verify(sink, times(tableNames.length)).initializeAndGetReadLatencyForTable(isA(String.class));
	for (int i=0; i<2; i++) {
	assertNotEquals("verify non-null read latency", null, sink.getReadLatencyMap().get(tableNames[i].getNameAsString()));
	assertNotEquals("verify non-zero read latency", 0L, sink.getReadLatencyMap().get(tableNames[i].getNameAsString()));
	}
	// One table's timeout is set for 0 ms and thus, should lead to an error.
	verify(mockAppender, times(1)).doAppend(argThat(new ArgumentMatcher<LoggingEvent>() {
	@Override
	public boolean matches(LoggingEvent argument) {
	return ((LoggingEvent) argument).getRenderedMessage().contains("exceeded the configured read timeout.");
	}
	}));
	verify(mockAppender, times(2)).doAppend(argThat(new ArgumentMatcher<LoggingEvent>() {
	@Override
	public boolean matches(LoggingEvent argument) {
	return argument.getRenderedMessage().contains("Configured read timeout");
	}
	}));
	}

	@Test
	public void testWriteTableTimeout() throws Exception {
	ExecutorService executor = new ScheduledThreadPoolExecutor(1);
	CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
	CanaryTool canary = new CanaryTool(executor, sink);
	String[] args = { "-writeSniffing", "-writeTableTimeout", String.valueOf(Long.MAX_VALUE)};
	assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
	assertNotEquals("verify non-null write latency", null, sink.getWriteLatency());
	assertNotEquals("verify non-zero write latency", 0L, sink.getWriteLatency());
	verify(mockAppender, times(1)).doAppend(argThat(
	new ArgumentMatcher<LoggingEvent>() {
	@Override
	public boolean matches(LoggingEvent argument) {
	return argument.getRenderedMessage().contains("Configured write timeout");
	}
	}));
	}

	//no table created, so there should be no regions
	@Test
	public void testRegionserverNoRegions() throws Exception {
	runRegionserverCanary();
	verify(mockAppender).doAppend(argThat(new ArgumentMatcher<LoggingEvent>() {
	@Override
	public boolean matches(LoggingEvent argument) {
	return argument.getRenderedMessage().contains("Regionserver not serving any regions");
	}
	}));
	}

	//by creating a table, there shouldn't be any region servers not serving any regions
	@Test
	public void testRegionserverWithRegions() throws Exception {
	final TableName tableName = TableName.valueOf(name.getMethodName());
	testingUtility.createTable(tableName, new byte[][] { FAMILY });
	runRegionserverCanary();
	verify(mockAppender, never()).doAppend(argThat(new ArgumentMatcher<LoggingEvent>() {
	@Override
	public boolean matches(LoggingEvent argument) {
	return argument.getRenderedMessage().contains("Regionserver not serving any regions");
	}
	}));
	}

	@Test
	public void testRawScanConfig() throws Exception {
	final TableName tableName = TableName.valueOf(name.getMethodName());
	Table table = testingUtility.createTable(tableName, new byte[][] { FAMILY });
	// insert some test rows
	for (int i=0; i<1000; i++) {
	byte[] iBytes = Bytes.toBytes(i);
	Put p = new Put(iBytes);
	p.addColumn(FAMILY, COLUMN, iBytes);
	table.put(p);
	}
	ExecutorService executor = new ScheduledThreadPoolExecutor(1);
	CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
	CanaryTool canary = new CanaryTool(executor, sink);
	String[] args = { "-t", "10000", name.getMethodName() };
	org.apache.hadoop.conf.Configuration conf =
	new org.apache.hadoop.conf.Configuration(testingUtility.getConfiguration());
	conf.setBoolean(HConstants.HBASE_CANARY_READ_RAW_SCAN_KEY, true);
	assertEquals(0, ToolRunner.run(conf, canary, args));
	verify(sink, atLeastOnce())
	.publishReadTiming(isA(ServerName.class), isA(RegionInfo.class),
	isA(ColumnFamilyDescriptor.class), anyLong());
	assertEquals("verify no read error count", 0, canary.getReadFailures().size());
	}

	private void runRegionserverCanary() throws Exception {
	ExecutorService executor = new ScheduledThreadPoolExecutor(1);
	CanaryTool canary = new CanaryTool(executor, new CanaryTool.RegionServerStdOutSink());
	String[] args = { "-t", "10000", "-regionserver"};
	assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
	assertEquals("verify no read error count", 0, canary.getReadFailures().size());
	}

	private void testZookeeperCanaryWithArgs(String[] args) throws Exception {
	Integer port =
	Iterables.getOnlyElement(testingUtility.getZkCluster().getClientPortList(), null);
	testingUtility.getConfiguration().set(HConstants.ZOOKEEPER_QUORUM, "localhost:" + port);
	ExecutorService executor = new ScheduledThreadPoolExecutor(2);
	CanaryTool.ZookeeperStdOutSink sink = spy(new CanaryTool.ZookeeperStdOutSink());
	CanaryTool canary = new CanaryTool(executor, sink);
	assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));

	String baseZnode = testingUtility.getConfiguration()
	.get(HConstants.ZOOKEEPER_ZNODE_PARENT, HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
	verify(sink, atLeastOnce())
	.publishReadTiming(eq(baseZnode), eq("localhost:" + port), anyLong());
	}
	}