hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatBase.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hbase.mapreduce;

import static org.junit.Assert.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

import java.io.IOException;
import java.net.Inet6Address;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.ExecutorService;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.AsyncConnection;
import org.apache.hadoop.hbase.client.BufferedMutator;
import org.apache.hadoop.hbase.client.BufferedMutatorParams;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionUtils;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableBuilder;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.mapreduce.JobContext;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;

@Category({SmallTests.class})
public class TestTableInputFormatBase {

  @ClassRule
  public static final HBaseClassTestRule CLASS_RULE =
      HBaseClassTestRule.forClass(TestTableInputFormatBase.class);

  @Test
  public void testTableInputFormatBaseReverseDNSForIPv6()
      throws UnknownHostException {
    String address = "ipv6.google.com";
    String localhost = null;
    InetAddress addr = null;
    TableInputFormat inputFormat = new TableInputFormat();
    try {
      localhost = InetAddress.getByName(address).getCanonicalHostName();
      addr = Inet6Address.getByName(address);
    } catch (UnknownHostException e) {
      // google.com is down, we can probably forgive this test.
      return;
    }
    System.out.println("Should retrun the hostname for this host " +
        localhost + " addr : " + addr);
    String actualHostName = inputFormat.reverseDNS(addr);
    assertEquals("Should retrun the hostname for this host. Expected : " +
        localhost + " Actual : " + actualHostName, localhost, actualHostName);
  }

  @Test
  public void testNonSuccessiveSplitsAreNotMerged() throws IOException {
    JobContext context = mock(JobContext.class);
    Configuration conf = HBaseConfiguration.create();
    conf.set(ConnectionUtils.HBASE_CLIENT_CONNECTION_IMPL,
        ConnectionForMergeTesting.class.getName());
    conf.set(TableInputFormat.INPUT_TABLE, "testTable");
    conf.setBoolean(TableInputFormatBase.MAPREDUCE_INPUT_AUTOBALANCE, true);
    when(context.getConfiguration()).thenReturn(conf);

    TableInputFormat tifExclude = new TableInputFormatForMergeTesting();
    tifExclude.setConf(conf);
    // split["b", "c"] is excluded, split["o", "p"] and split["p", "q"] are merged,
    // but split["a", "b"] and split["c", "d"] are not merged.
    assertEquals(ConnectionForMergeTesting.START_KEYS.length - 1 - 1,
        tifExclude.getSplits(context).size());
  }

  /**
   * Subclass of {@link TableInputFormat} to use in {@link #testNonSuccessiveSplitsAreNotMerged}.
   * This class overrides {@link TableInputFormatBase#includeRegionInSplit}
   * to exclude specific splits.
   */
  private static class TableInputFormatForMergeTesting extends TableInputFormat {
    private byte[] prefixStartKey = Bytes.toBytes("b");
    private byte[] prefixEndKey = Bytes.toBytes("c");
    private RegionSizeCalculator sizeCalculator;

    /**
     * Exclude regions which contain rows starting with "b".
     */
    @Override
    protected boolean includeRegionInSplit(final byte[] startKey, final byte [] endKey) {
      if (Bytes.compareTo(startKey, prefixEndKey) < 0
          && (Bytes.compareTo(prefixStartKey, endKey) < 0
              || Bytes.equals(endKey, HConstants.EMPTY_END_ROW))) {
        return false;
      } else {
        return true;
      }
    }

    @Override
    protected void initializeTable(Connection connection, TableName tableName) throws IOException {
      super.initializeTable(connection, tableName);
      ConnectionForMergeTesting cft = (ConnectionForMergeTesting) connection;
      sizeCalculator = cft.getRegionSizeCalculator();
    }

    @Override
    protected RegionSizeCalculator createRegionSizeCalculator(RegionLocator locator, Admin admin)
      throws IOException {
      return sizeCalculator;
    }
  }

  /**
   * Connection class to use in {@link #testNonSuccessiveSplitsAreNotMerged}.
   * This class returns mocked {@link Table}, {@link RegionLocator}, {@link RegionSizeCalculator},
   * and {@link Admin}.
   */
  private static class ConnectionForMergeTesting implements Connection {
    public static final byte[][] SPLITS = new byte[][] {
      Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c"), Bytes.toBytes("d"),
      Bytes.toBytes("e"), Bytes.toBytes("f"), Bytes.toBytes("g"), Bytes.toBytes("h"),
      Bytes.toBytes("i"), Bytes.toBytes("j"), Bytes.toBytes("k"), Bytes.toBytes("l"),
      Bytes.toBytes("m"), Bytes.toBytes("n"), Bytes.toBytes("o"), Bytes.toBytes("p"),
      Bytes.toBytes("q"), Bytes.toBytes("r"), Bytes.toBytes("s"), Bytes.toBytes("t"),
      Bytes.toBytes("u"), Bytes.toBytes("v"), Bytes.toBytes("w"), Bytes.toBytes("x"),
      Bytes.toBytes("y"), Bytes.toBytes("z")
    };

    public static final byte[][] START_KEYS;
    public static final byte[][] END_KEYS;
    static {
      START_KEYS = new byte[SPLITS.length + 1][];
      START_KEYS[0] = HConstants.EMPTY_BYTE_ARRAY;
      for (int i = 0; i < SPLITS.length; i++) {
        START_KEYS[i + 1] = SPLITS[i];
      }

      END_KEYS = new byte[SPLITS.length + 1][];
      for (int i = 0; i < SPLITS.length; i++) {
        END_KEYS[i] = SPLITS[i];
      }
      END_KEYS[SPLITS.length] = HConstants.EMPTY_BYTE_ARRAY;
    }

    public static final Map<byte[], Long> SIZE_MAP = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    static {
      for (byte[] startKey : START_KEYS) {
        SIZE_MAP.put(startKey, 1024L * 1024L * 1024L);
      }
      SIZE_MAP.put(Bytes.toBytes("a"), 200L * 1024L * 1024L);
      SIZE_MAP.put(Bytes.toBytes("b"), 200L * 1024L * 1024L);
      SIZE_MAP.put(Bytes.toBytes("c"), 200L * 1024L * 1024L);
      SIZE_MAP.put(Bytes.toBytes("o"), 200L * 1024L * 1024L);
      SIZE_MAP.put(Bytes.toBytes("p"), 200L * 1024L * 1024L);
    }

    ConnectionForMergeTesting(Configuration conf, ExecutorService pool, User user)
        throws IOException {
    }

    @Override
    public void abort(String why, Throwable e) {
    }

    @Override
    public boolean isAborted() {
      return false;
    }

    @Override
    public Configuration getConfiguration() {
      throw new UnsupportedOperationException();
    }

    @Override
    public Table getTable(TableName tableName) throws IOException {
      Table table = mock(Table.class);
      when(table.getName()).thenReturn(tableName);
      return table;
    }

    @Override
    public Table getTable(TableName tableName, ExecutorService pool) throws IOException {
      throw new UnsupportedOperationException();
    }

    @Override
    public BufferedMutator getBufferedMutator(TableName tableName) throws IOException {
      throw new UnsupportedOperationException();
    }

    @Override
    public BufferedMutator getBufferedMutator(BufferedMutatorParams params) throws IOException {
      throw new UnsupportedOperationException();
    }

    @Override
    public RegionLocator getRegionLocator(TableName tableName) throws IOException {
      final Map<byte[], HRegionLocation> locationMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
      for (byte[] startKey : START_KEYS) {
        HRegionLocation hrl = new HRegionLocation(
            RegionInfoBuilder.newBuilder(tableName).setStartKey(startKey).build(),
            ServerName.valueOf("localhost", 0, 0));
        locationMap.put(startKey, hrl);
      }

      RegionLocator locator = mock(RegionLocator.class);
      when(locator.getRegionLocation(any(byte [].class), anyBoolean())).
        thenAnswer(new Answer<HRegionLocation>() {
          @Override
          public HRegionLocation answer(InvocationOnMock invocationOnMock) throws Throwable {
            Object [] args = invocationOnMock.getArguments();
            byte [] key = (byte [])args[0];
            return locationMap.get(key);
          }
        });
      when(locator.getStartEndKeys()).
        thenReturn(new Pair<byte[][], byte[][]>(START_KEYS, END_KEYS));
      return locator;
    }

    public RegionSizeCalculator getRegionSizeCalculator() {
      RegionSizeCalculator sizeCalculator = mock(RegionSizeCalculator.class);
      when(sizeCalculator.getRegionSize(any(byte[].class))).
        thenAnswer(new Answer<Long>() {
          @Override
          public Long answer(InvocationOnMock invocationOnMock) throws Throwable {
            Object [] args = invocationOnMock.getArguments();
            byte [] regionId = (byte [])args[0];
            byte[] startKey = RegionInfo.getStartKey(regionId);
            return SIZE_MAP.get(startKey);
          }
        });
      return sizeCalculator;
    }

    @Override
    public Admin getAdmin() throws IOException {
      Admin admin = mock(Admin.class);
      // return non-null admin to pass null checks
      return admin;
    }

    @Override
    public void close() throws IOException {
    }

    @Override
    public boolean isClosed() {
      return false;
    }

    @Override
    public TableBuilder getTableBuilder(TableName tableName, ExecutorService pool) {
      throw new UnsupportedOperationException();
    }

    @Override
    public void clearRegionLocationCache() {
    }

    @Override
    public AsyncConnection toAsyncConnection() {
      throw new UnsupportedOperationException();
    }

    @Override
    public String getClusterId() {
      return null;
    }
  }
}