tephra-hbase-compat-2.4/src/test/java/org/apache/tephra/hbase/txprune/DataJanitorStateTest.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.tephra.hbase.txprune;


import com.google.common.collect.ImmutableSortedMap;
import com.google.common.collect.ImmutableSortedSet;

import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.tephra.TxConstants;
import org.apache.tephra.hbase.AbstractHBaseTableTest;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.util.Collections;
import java.util.Map;
import java.util.SortedSet;
import java.util.TreeMap;
import java.util.TreeSet;

/**
 * Test methods of {@link DataJanitorState}
 */
// TODO: Group all the tests that need HBase mini cluster into a suite, so that we start the mini-cluster only once
public class DataJanitorStateTest extends AbstractHBaseTableTest {

  private TableName pruneStateTable;
  private DataJanitorState dataJanitorState;

  @Before
  public void beforeTest() throws Exception {
    pruneStateTable = TableName.valueOf(conf.get(TxConstants.TransactionPruning.PRUNE_STATE_TABLE,
                                                 TxConstants.TransactionPruning.DEFAULT_PRUNE_STATE_TABLE));
    Table table = createTable(pruneStateTable.getName(), new byte[][]{DataJanitorState.FAMILY}, false,
                               // Prune state table is a non-transactional table, hence no transaction co-processor
                               Collections.<String>emptyList());
    table.close();

    dataJanitorState =
      new DataJanitorState(new DataJanitorState.TableSupplier() {
        @Override
        public Table get() throws IOException {
          return testUtil.getConnection().getTable(pruneStateTable);
        }
      });

  }

  @After
  public void afterTest() throws Exception {
    hBaseAdmin.disableTable(pruneStateTable);
    hBaseAdmin.deleteTable(pruneStateTable);
  }

  @Test
  public void testSavePruneUpperBound() throws Exception {
    int max = 20;

    // Nothing should be present in the beginning
    Assert.assertEquals(-1, dataJanitorState.getPruneUpperBoundForRegion(Bytes.toBytes(10L)));

    // Save some region - prune upper bound values
    // We should have values for regions 0, 2, 4, 6, ..., max-2 after this
    for (long i = 0; i < max; i += 2) {
      dataJanitorState.savePruneUpperBoundForRegion(Bytes.toBytes(i), i);
    }

    Assert.assertEquals(10L, dataJanitorState.getPruneUpperBoundForRegion(Bytes.toBytes(10L)));

    // Verify all the saved values
    for (long i = 0; i < max; ++i) {
      long expected = i % 2 == 0 ? i : -1;
      Assert.assertEquals(expected, dataJanitorState.getPruneUpperBoundForRegion(Bytes.toBytes(i)));
    }
    // Regions not present should give -1
    Assert.assertEquals(-1, dataJanitorState.getPruneUpperBoundForRegion(Bytes.toBytes(max + 50L)));
    Assert.assertEquals(-1, dataJanitorState.getPruneUpperBoundForRegion(Bytes.toBytes((max + 10L) * -1)));
    Assert.assertEquals(-1, dataJanitorState.getPruneUpperBoundForRegion(Bytes.toBytes(3L)));

    SortedSet<byte[]> allRegions = new TreeSet<>(Bytes.BYTES_COMPARATOR);
    Map<byte[], Long> expectedMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    for (long i = 0; i < max; ++i) {
      allRegions.add(Bytes.toBytes(i));
      if (i % 2 == 0) {
        expectedMap.put(Bytes.toBytes(i), i);
      }
    }
    Assert.assertEquals(max / 2, expectedMap.size());
    Assert.assertEquals(expectedMap, dataJanitorState.getPruneUpperBoundForRegions(allRegions));

    SortedSet<byte[]> regions = ImmutableSortedSet.orderedBy(Bytes.BYTES_COMPARATOR)
      .add(Bytes.toBytes((max + 20L) * -1))
      .add(Bytes.toBytes(6L))
      .add(Bytes.toBytes(15L))
      .add(Bytes.toBytes(18L))
      .add(Bytes.toBytes(max + 33L))
      .build();
    expectedMap = ImmutableSortedMap.<byte[], Long>orderedBy(Bytes.BYTES_COMPARATOR)
      .put(Bytes.toBytes(6L), 6L)
      .put(Bytes.toBytes(18L), 18L)
      .build();
    Assert.assertEquals(expectedMap, dataJanitorState.getPruneUpperBoundForRegions(regions));

    // Delete regions that have prune upper bound before 15 and not in set (4, 8)
    ImmutableSortedSet<byte[]> excludeRegions =
      ImmutableSortedSet.orderedBy(Bytes.BYTES_COMPARATOR).add(Bytes.toBytes(4L)).add(Bytes.toBytes(8L)).build();
    dataJanitorState.deletePruneUpperBounds(15, excludeRegions);
    // Regions 0, 2, 6 and 10 should have been deleted now
    expectedMap = ImmutableSortedMap.<byte[], Long>orderedBy(Bytes.BYTES_COMPARATOR)
      .put(Bytes.toBytes(4L), 4L)
      .put(Bytes.toBytes(8L), 8L)
      .put(Bytes.toBytes(16L), 16L)
      .put(Bytes.toBytes(18L), 18L)
      .build();
    Assert.assertEquals(expectedMap, dataJanitorState.getPruneUpperBoundForRegions(allRegions));
  }

  @Test(timeout = 30000L) // The timeout is used to verify the fix for TEPHRA-230, the test will timeout without the fix
  public void testSaveRegionTime() throws Exception {
    int maxTime = 100;

    // Nothing should be present in the beginning
    Assert.assertNull(dataJanitorState.getRegionsOnOrBeforeTime(maxTime));

    // Save regions for time
    Map<Long, SortedSet<byte[]>> regionsTime = new TreeMap<>();
    for (long time = 0; time < maxTime; time += 10) {
      SortedSet<byte[]> regions = new TreeSet<>(Bytes.BYTES_COMPARATOR);
      for (long region = 0; region < 10; region += 2) {
        regions.add(Bytes.toBytes((time * 10) + region));
      }
      regionsTime.put(time, regions);
      dataJanitorState.saveRegionsForTime(time, regions);
    }

    // Verify saved regions
    Assert.assertEquals(new TimeRegions(0, regionsTime.get(0L)), dataJanitorState.getRegionsOnOrBeforeTime(0));
    Assert.assertEquals(new TimeRegions(30, regionsTime.get(30L)), dataJanitorState.getRegionsOnOrBeforeTime(30));
    Assert.assertEquals(new TimeRegions(20, regionsTime.get(20L)), dataJanitorState.getRegionsOnOrBeforeTime(25));
    Assert.assertEquals(new TimeRegions(30, regionsTime.get(30L)), dataJanitorState.getRegionsOnOrBeforeTime(31));
    Assert.assertEquals(new TimeRegions(90, regionsTime.get(90L)),
                        dataJanitorState.getRegionsOnOrBeforeTime(maxTime + 1000));
    Assert.assertNull(dataJanitorState.getRegionsOnOrBeforeTime(-10));

    // Now change the count stored for regions saved at time 0, 30 and 90
    try (Table stateTable = testUtil.getConnection().getTable(pruneStateTable)) {
      dataJanitorState.saveRegionCountForTime(stateTable, Bytes.toBytes(Long.MAX_VALUE), 3);
      dataJanitorState.saveRegionCountForTime(stateTable, Bytes.toBytes(Long.MAX_VALUE - 30L), 3);
      dataJanitorState.saveRegionCountForTime(stateTable, Bytes.toBytes(Long.MAX_VALUE - 90L), 0);
    }

    // Now querying for time 0 should return null, and querying for time 30 should return regions from time 20
    Assert.assertNull(dataJanitorState.getRegionsOnOrBeforeTime(0));
    Assert.assertEquals(new TimeRegions(20, regionsTime.get(20L)), dataJanitorState.getRegionsOnOrBeforeTime(30));
    Assert.assertEquals(new TimeRegions(20, regionsTime.get(20L)), dataJanitorState.getRegionsOnOrBeforeTime(35));
    Assert.assertEquals(new TimeRegions(20, regionsTime.get(20L)), dataJanitorState.getRegionsOnOrBeforeTime(25));
    // Querying for anything higher than 90 should give 80 (reproduces TEPHRA-230)
    Assert.assertEquals(new TimeRegions(80, regionsTime.get(80L)),
                        dataJanitorState.getRegionsOnOrBeforeTime(Long.MAX_VALUE));

    // Delete regions saved on or before time 30
    dataJanitorState.deleteAllRegionsOnOrBeforeTime(30);
    // Values on or before time 30 should be deleted
    Assert.assertNull(dataJanitorState.getRegionsOnOrBeforeTime(30));
    Assert.assertNull(dataJanitorState.getRegionsOnOrBeforeTime(25));
    // Counts should be deleted for time on or before 30
    try (Table stateTable = testUtil.getConnection().getTable(pruneStateTable)) {
      Assert.assertEquals(-1, dataJanitorState.getRegionCountForTime(stateTable, 30));
      Assert.assertEquals(-1, dataJanitorState.getRegionCountForTime(stateTable, 0));
    }
    // Values after time 30 should still exist
    Assert.assertEquals(new TimeRegions(40, regionsTime.get(40L)), dataJanitorState.getRegionsOnOrBeforeTime(40));
    try (Table stateTable = testUtil.getConnection().getTable(pruneStateTable)) {
      Assert.assertEquals(5, dataJanitorState.getRegionCountForTime(stateTable, 40));
    }
  }

  @Test
  public void testSaveInactiveTransactionBoundTime() throws Exception {
    int maxTime = 100;

    // Nothing should be present in the beginning
    Assert.assertEquals(-1, dataJanitorState.getInactiveTransactionBoundForTime(10));

    // Save inactive transaction bounds for various time values
    for (long time = 0; time < maxTime; time += 10) {
      dataJanitorState.saveInactiveTransactionBoundForTime(time, time + 2);
    }

    // Verify written values
    Assert.assertEquals(2, dataJanitorState.getInactiveTransactionBoundForTime(0));
    Assert.assertEquals(12, dataJanitorState.getInactiveTransactionBoundForTime(10));
    Assert.assertEquals(-1, dataJanitorState.getInactiveTransactionBoundForTime(15));
    Assert.assertEquals(92, dataJanitorState.getInactiveTransactionBoundForTime(90));
    Assert.assertEquals(-1, dataJanitorState.getInactiveTransactionBoundForTime(maxTime + 100));
    Assert.assertEquals(-1, dataJanitorState.getInactiveTransactionBoundForTime((maxTime + 55) * -1L));

    // Delete values saved on or before time 20
    dataJanitorState.deleteInactiveTransactionBoundsOnOrBeforeTime(20);
    // Values on or before time 20 should be deleted
    Assert.assertEquals(-1, dataJanitorState.getInactiveTransactionBoundForTime(0));
    Assert.assertEquals(-1, dataJanitorState.getInactiveTransactionBoundForTime(10));
    Assert.assertEquals(-1, dataJanitorState.getInactiveTransactionBoundForTime(20));
    // Values after time 20 should still exist
    Assert.assertEquals(32, dataJanitorState.getInactiveTransactionBoundForTime(30));
    Assert.assertEquals(92, dataJanitorState.getInactiveTransactionBoundForTime(90));
  }

  @Test
  public void testSaveEmptyRegions() throws Exception {
    // Nothing should be present in the beginning
    Assert.assertEquals(ImmutableSortedSet.<byte[]>of(), dataJanitorState.getEmptyRegionsAfterTime(-1, null));

    byte[] region1 = Bytes.toBytes("region1");
    byte[] region2 = Bytes.toBytes("region2");
    byte[] region3 = Bytes.toBytes("region3");
    byte[] region4 = Bytes.toBytes("region4");
    SortedSet<byte[]> allRegions = toISet(region1, region2, region3, region4);

    // Now record some empty regions
    dataJanitorState.saveEmptyRegionForTime(100, region1);
    dataJanitorState.saveEmptyRegionForTime(110, region1);
    dataJanitorState.saveEmptyRegionForTime(102, region2);
    dataJanitorState.saveEmptyRegionForTime(112, region3);

    Assert.assertEquals(toISet(region1, region2, region3),
                        dataJanitorState.getEmptyRegionsAfterTime(-1, null));

    Assert.assertEquals(toISet(region1, region2, region3),
                        dataJanitorState.getEmptyRegionsAfterTime(100, allRegions));

    Assert.assertEquals(toISet(region2, region3),
                        dataJanitorState.getEmptyRegionsAfterTime(100, toISet(region2, region3)));

    Assert.assertEquals(toISet(),
                        dataJanitorState.getEmptyRegionsAfterTime(100, ImmutableSortedSet.<byte[]>of()));

    Assert.assertEquals(toISet(region3),
                        dataJanitorState.getEmptyRegionsAfterTime(110, allRegions));

    Assert.assertEquals(toISet(),
                        dataJanitorState.getEmptyRegionsAfterTime(112, allRegions));

    // Delete empty regions on or before time 110
    dataJanitorState.deleteEmptyRegionsOnOrBeforeTime(110);
    // Now only region3 should remain
    Assert.assertEquals(toISet(region3), dataJanitorState.getEmptyRegionsAfterTime(-1, null));
    Assert.assertEquals(toISet(region3), dataJanitorState.getEmptyRegionsAfterTime(100, allRegions));

    // Delete empty regions on or before time 150
    dataJanitorState.deleteEmptyRegionsOnOrBeforeTime(150);
    // Now nothing should remain
    Assert.assertEquals(toISet(), dataJanitorState.getEmptyRegionsAfterTime(-1, null));
  }

  private ImmutableSortedSet<byte[]> toISet(byte[]... args) {
    ImmutableSortedSet.Builder<byte[]> builder = ImmutableSortedSet.orderedBy(Bytes.BYTES_COMPARATOR);
    for (byte[] arg : args) {
      builder.add(arg);
    }
    return builder.build();
  }
}