| /* |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.pig.backend.hadoop.hbase; |
| |
| import org.apache.commons.cli.ParseException; |
| import org.apache.hadoop.hbase.client.Scan; |
| import org.apache.hadoop.hbase.filter.ColumnPrefixFilter; |
| import org.apache.hadoop.hbase.filter.CompareFilter; |
| import org.apache.hadoop.hbase.filter.Filter; |
| import org.apache.hadoop.hbase.filter.FilterList; |
| import org.apache.hadoop.hbase.filter.FamilyFilter; |
| import org.apache.hadoop.hbase.filter.QualifierFilter; |
| import org.apache.hadoop.hbase.util.Bytes; |
| import org.apache.pig.impl.util.UDFContext; |
| import org.junit.Test; |
| |
| import java.io.IOException; |
| import java.lang.reflect.Field; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.Properties; |
| |
| import static org.junit.Assert.assertEquals; |
| import static org.junit.Assert.assertNull; |
| import static org.junit.Assert.assertNotNull; |
| import static org.junit.Assert.assertTrue; |
| |
| /** |
| * Class to test how filters are used by HBaseStorage, since mis-use can be costly in terms of |
| * perfomance. |
| */ |
| public class TestHBaseStorageFiltering { |
| |
| private Filter getHBaseStorageFilter(String firstArg) |
| throws NoSuchFieldException, ParseException, IOException, IllegalAccessException { |
| return initHBaseStorageFilter(firstArg, ""); |
| } |
| |
| // Helper to initialize HBaseStorage and return its Filter |
| private Filter initHBaseStorageFilter(String firstArg, String secondArg) |
| throws NoSuchFieldException, ParseException, IOException, IllegalAccessException { |
| UDFContext.getUDFContext().setClientSystemProps(new Properties()); |
| |
| Field scanField = HBaseStorage.class.getDeclaredField("scan"); |
| scanField.setAccessible(true); |
| |
| HBaseStorage storage = new HBaseStorage(firstArg, secondArg); |
| Scan scan = (Scan)scanField.get(storage); |
| return scan.getFilter(); |
| } |
| |
| // Helper to initialize HBaseStorage and return its List of ColumnInfo objects |
| @SuppressWarnings("unchecked") |
| private List<HBaseStorage.ColumnInfo> getHBaseColumnInfo(String firstArg, String secondArg) |
| throws NoSuchFieldException, ParseException, IOException, IllegalAccessException { |
| UDFContext.getUDFContext().setClientSystemProps(new Properties()); |
| |
| Field columnInfoField = HBaseStorage.class.getDeclaredField("columnInfo_"); |
| columnInfoField.setAccessible(true); |
| |
| HBaseStorage storage = new HBaseStorage(firstArg, secondArg); |
| List<HBaseStorage.ColumnInfo> columnInfoList = |
| (List<HBaseStorage.ColumnInfo>)columnInfoField.get(storage); |
| return columnInfoList; |
| } |
| |
| @Test |
| public void testDescriptors() throws Exception { |
| Filter filter = getHBaseStorageFilter("cf1:a cf1:b"); |
| assertNull(filter); |
| } |
| |
| @Test |
| public void testFamily() throws Exception { |
| Filter filter = getHBaseStorageFilter("cf1:"); |
| assertNull(filter); |
| } |
| |
| @Test |
| public void testFamily2() throws Exception { |
| Filter filter = getHBaseStorageFilter("cf1:*"); |
| assertNull(filter); |
| } |
| |
| @Test |
| public void testPrefix() throws Exception { |
| Filter filter = getHBaseStorageFilter("cf1:foo*"); |
| List<Filter> childFilters = assertFilterList(filter, FilterList.Operator.MUST_PASS_ALL, 1); |
| childFilters = assertFilterList(childFilters.get(0), FilterList.Operator.MUST_PASS_ONE, 1); |
| childFilters = assertFilterList(childFilters.get(0), FilterList.Operator.MUST_PASS_ALL, 2); |
| assertFamilyFilter(childFilters.get(0), CompareFilter.CompareOp.EQUAL, "cf1"); |
| childFilters = assertFilterList(childFilters.get(1), FilterList.Operator.MUST_PASS_ONE, 1); |
| assertPrefixFilter(childFilters.get(0), "foo"); |
| } |
| |
| @Test |
| public void testDescriptorsAndFamily1() throws Exception { |
| Filter filter = getHBaseStorageFilter("cf1:a cf1:b cf2:"); |
| assertNull(filter); |
| } |
| |
| @Test |
| public void testDescriptorsAndFamily2() throws Exception { |
| Filter filter = getHBaseStorageFilter("cf1:a cf1:b cf2:*"); |
| assertNull(filter); |
| } |
| |
| @Test |
| public void testDescriptorsAndPrefix() throws Exception { |
| Filter filter = getHBaseStorageFilter("cf1:a cf1:b cf2:foo*"); |
| List<Filter> childFilters = assertFilterList(filter, FilterList.Operator.MUST_PASS_ALL, 1); |
| List<Filter> groupFilters = assertFilterList(childFilters.get(0), FilterList.Operator.MUST_PASS_ONE, 2); |
| |
| List<Filter> firstFilters = assertFilterList(groupFilters.get(0), FilterList.Operator.MUST_PASS_ALL, 2); |
| FamilyFilter firstFamilyFilter = assertFamilyFilter(firstFilters.get(0), CompareFilter.CompareOp.EQUAL); |
| |
| List<Filter> secondFilters = assertFilterList(groupFilters.get(1), FilterList.Operator.MUST_PASS_ALL, 2); |
| FamilyFilter secondFamilyFilter = assertFamilyFilter(secondFilters.get(0), CompareFilter.CompareOp.EQUAL); |
| |
| // one of the above will be the cf1 filters, one will be the cf2. Order is unknown |
| Filter cf1ColumnList; |
| Filter cf2ColumnList; |
| if (Bytes.toString(firstFamilyFilter.getComparator().getValue()).equals("cf1")) { |
| assertEquals("cf2", Bytes.toString(secondFamilyFilter.getComparator().getValue())); |
| cf1ColumnList = firstFilters.get(1); |
| cf2ColumnList = secondFilters.get(1); |
| } |
| else { |
| assertEquals("cf1", Bytes.toString(secondFamilyFilter.getComparator().getValue())); |
| assertEquals("cf2", Bytes.toString(firstFamilyFilter.getComparator().getValue())); |
| cf1ColumnList = secondFilters.get(1); |
| cf2ColumnList = firstFilters.get(1); |
| } |
| |
| List<Filter> c1ColumnFilters = assertFilterList(cf1ColumnList, FilterList.Operator.MUST_PASS_ONE, 2); |
| assertQualifierFilter(c1ColumnFilters.get(0), CompareFilter.CompareOp.EQUAL, "a"); |
| assertQualifierFilter(c1ColumnFilters.get(1), CompareFilter.CompareOp.EQUAL, "b"); |
| |
| List<Filter> c2ColumnFilters = assertFilterList(cf2ColumnList, FilterList.Operator.MUST_PASS_ONE, 1); |
| assertPrefixFilter(c2ColumnFilters.get(0), "foo"); |
| } |
| |
| @Test |
| public void testColumnGroups() throws Exception { |
| List<HBaseStorage.ColumnInfo> columnInfoList = getHBaseColumnInfo("cf1:a cf1:b cf2:foo*", ""); |
| Map<String, List<HBaseStorage.ColumnInfo>> groupMap = HBaseStorage.groupByFamily(columnInfoList); |
| assertEquals(2, groupMap.size()); |
| |
| List<HBaseStorage.ColumnInfo> cf1List = groupMap.get("cf1"); |
| assertNotNull(cf1List); |
| assertEquals(2, cf1List.size()); |
| assertColumnInfo(cf1List.get(0), "cf1", "a", null); |
| assertColumnInfo(cf1List.get(1), "cf1", "b", null); |
| |
| List<HBaseStorage.ColumnInfo> cf2List = groupMap.get("cf2"); |
| assertNotNull(cf2List); |
| assertEquals(1, cf2List.size()); |
| assertColumnInfo(cf2List.get(0), "cf2", null, "foo"); |
| } |
| |
| @Test |
| public void testIncrementStrings() { |
| doIncrementTest("100", "101"); |
| doIncrementTest("0001", "0002"); |
| doIncrementTest("aaaccccc", "aaaccccd"); |
| } |
| |
| @Test |
| public void testIncrementBytes() { |
| doIncrementTest(Bytes.toBytes(0x03), Bytes.toBytes(0x04)); |
| doIncrementTest(new byte[] {0, 1, 0}, new byte[] {0, 1, 1}); |
| doIncrementTest(new byte[] {127}, new byte[] {-128}); |
| doIncrementTest(new byte[] {-1}, new byte[] {-1, 0}); |
| doIncrementTest(new byte[] {-1, -1}, new byte[] {-1, -1, 0}); |
| doIncrementTest(new byte[] {0, -1, -1}, new byte[] {1, 0, 0}); |
| doIncrementTest(Bytes.toBytes(0xFFFFFFFF), new byte[] {-1, -1, -1, -1, 0}); |
| doIncrementTest(Bytes.toBytes(Long.MAX_VALUE), new byte[] {-128, 0, 0, 0, 0, 0, 0, 0}); |
| } |
| |
| private void doIncrementTest(String initial, String expected) { |
| byte[] initialBytes = Bytes.toBytes(initial); |
| byte[] expectedBytes = Bytes.toBytes(expected); |
| byte[] incrementedBytes = HBaseStorage.increment(initialBytes); |
| assertTrue(String.format("Expected bytes %s (%s) did not equal found bytes %s (%s)", |
| HBaseStorage.toString(expectedBytes), expected, HBaseStorage.toString(incrementedBytes), Bytes.toString(incrementedBytes)), |
| Bytes.compareTo(expectedBytes, incrementedBytes) == 0); |
| assertEquals(expected, Bytes.toString(incrementedBytes)); |
| assertTrue("Initial value of " + initial + " should be < " + Bytes.toString(incrementedBytes), |
| Bytes.compareTo(initialBytes, incrementedBytes) < 0); |
| } |
| |
| private void doIncrementTest(byte[] initial, byte[] expected) { |
| byte[] incrementedBytes = HBaseStorage.increment(initial); |
| if (expected == null) { |
| assertNull("Expected null bytes but found " + HBaseStorage.toString(incrementedBytes), incrementedBytes); |
| } else { |
| assertTrue(String.format("Expected bytes %s did not equal found bytes %s", |
| HBaseStorage.toString(expected), HBaseStorage.toString(incrementedBytes)), |
| Bytes.compareTo(expected, incrementedBytes) == 0); |
| } |
| assertTrue("Initial value of should be < incremented value", |
| Bytes.compareTo(initial, incrementedBytes) < 0); |
| } |
| |
| private void assertColumnInfo(HBaseStorage.ColumnInfo columnInfo, |
| String columnFamily, String columnName, String prefix) { |
| assertEquals(columnFamily, Bytes.toString(columnInfo.getColumnFamily())); |
| assertEquals(columnName, Bytes.toString(columnInfo.getColumnName())); |
| assertEquals(prefix, Bytes.toString(columnInfo.getColumnPrefix())); |
| } |
| |
| private List<Filter> assertFilterList(Filter filter, FilterList.Operator operator, int size) { |
| assertTrue("Filter is not a FilterList: " + filter.getClass().getSimpleName(), |
| filter instanceof FilterList); |
| FilterList filterList = (FilterList)filter; |
| assertEquals("Unexpected operator", operator, filterList.getOperator()); |
| assertEquals("Unexpected filter list size", size, filterList.getFilters().size()); |
| return filterList.getFilters(); |
| } |
| |
| private FamilyFilter assertFamilyFilter(Filter filter, CompareFilter.CompareOp compareOp, String value) { |
| FamilyFilter familyFilter = assertFamilyFilter(filter, compareOp); |
| assertEquals("Unexpected value", value, Bytes.toString(familyFilter.getComparator().getValue())); |
| return familyFilter; |
| } |
| |
| private FamilyFilter assertFamilyFilter(Filter filter, CompareFilter.CompareOp compareOp) { |
| assertTrue("Filter is not a FamilyFilter: " + filter.getClass().getSimpleName(), |
| filter instanceof FamilyFilter); |
| FamilyFilter familyFilter = (FamilyFilter)filter; |
| assertEquals("Unexpected compareOp", compareOp, familyFilter.getOperator()); |
| return familyFilter; |
| } |
| |
| private void assertPrefixFilter(Filter filter, String prefix) { |
| assertTrue("Filter is not a ColumnPrefixFilter: " + filter.getClass().getSimpleName(), |
| filter instanceof ColumnPrefixFilter); |
| ColumnPrefixFilter familyFilter = (ColumnPrefixFilter)filter; |
| assertEquals("Unexpected prefix", prefix, Bytes.toString(familyFilter.getPrefix())); |
| } |
| |
| private void assertQualifierFilter(Filter filter, CompareFilter.CompareOp compareOp, String value) { |
| assertTrue("Filter is not a QualifierFilter: " + filter.getClass().getSimpleName(), |
| filter instanceof QualifierFilter); |
| QualifierFilter qualifierFilter = (QualifierFilter)filter; |
| assertEquals("Unexpected compareOp", compareOp, qualifierFilter.getOperator()); |
| assertEquals("Unexpected value", value, Bytes.toString(qualifierFilter.getComparator().getValue())); |
| } |
| |
| private void printFilters(List<Filter> filters) { |
| for (Filter child : filters) { |
| printFilters(child); |
| } |
| } |
| |
| private void printFilters(Filter filter) { |
| if (filter != null) { |
| System.out.println(filter.toString()); |
| if (filter instanceof FilterList) { |
| printFilters(((FilterList)filter).getFilters()); |
| } |
| } |
| } |
| } |