| /* |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package org.apache.kylin.storage.gtrecord; |
| |
| import static org.junit.Assert.assertEquals; |
| |
| import java.io.IOException; |
| //import java.math.BigDecimal; |
| import java.nio.ByteBuffer; |
| import java.util.Arrays; |
| import java.util.BitSet; |
| import java.util.List; |
| |
| import org.apache.kylin.common.util.ByteArray; |
| import org.apache.kylin.common.util.BytesSerializer; |
| //import org.apache.kylin.common.util.Dictionary; |
| import org.apache.kylin.common.util.ImmutableBitSet; |
| import org.apache.kylin.common.util.LocalFileMetadataTestCase; |
| //import org.apache.kylin.cube.gridtable.CubeCodeSystem; |
| //import org.apache.kylin.dict.NumberDictionaryForestBuilder; |
| //import org.apache.kylin.dict.StringBytesConverter; |
| ////import org.apache.kylin.dict.TrieDictionaryBuilder; |
| //import org.apache.kylin.dimension.DictionaryDimEnc; |
| //import org.apache.kylin.dimension.DimensionEncoding; |
| //import org.apache.kylin.gridtable.GTBuilder; |
| import org.apache.kylin.gridtable.GTFilterScanner.FilterResultCache; |
| import org.apache.kylin.gridtable.GTInfo; |
| //import org.apache.kylin.gridtable.GTInfo.Builder; |
| import org.apache.kylin.gridtable.GTRecord; |
| import org.apache.kylin.gridtable.GTScanRange; |
| import org.apache.kylin.gridtable.GTScanRequest; |
| import org.apache.kylin.gridtable.GTScanRequestBuilder; |
| import org.apache.kylin.gridtable.GTUtil; |
| import org.apache.kylin.gridtable.GridTable; |
| import org.apache.kylin.gridtable.IGTScanner; |
| //import org.apache.kylin.gridtable.memstore.GTSimpleMemStore; |
| //import org.apache.kylin.metadata.datatype.DataType; |
| import org.apache.kylin.metadata.filter.ColumnTupleFilter; |
| import org.apache.kylin.metadata.filter.CompareTupleFilter; |
| import org.apache.kylin.metadata.filter.ConstantTupleFilter; |
| import org.apache.kylin.metadata.filter.ExtractTupleFilter; |
| import org.apache.kylin.metadata.filter.LogicalTupleFilter; |
| import org.apache.kylin.metadata.filter.TupleFilter; |
| import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum; |
| import org.apache.kylin.metadata.model.TableDesc; |
| import org.apache.kylin.metadata.model.TblColRef; |
| import org.apache.kylin.metadata.model.TblColRef.InnerDataTypeEnum; |
| import org.junit.After; |
| import org.junit.Assert; |
| import org.junit.Before; |
| import org.junit.Test; |
| import org.junit.Ignore; |
| |
| import org.apache.kylin.shaded.com.google.common.collect.Lists; |
| |
| @Ignore |
| public class DictGridTableTest extends LocalFileMetadataTestCase { |
| |
| private GridTable table; |
| private GTInfo info; |
| private CompareTupleFilter timeComp0; |
| private CompareTupleFilter timeComp1; |
| private CompareTupleFilter timeComp2; |
| private CompareTupleFilter timeComp3; |
| private CompareTupleFilter timeComp4; |
| private CompareTupleFilter timeComp5; |
| private CompareTupleFilter timeComp6; |
| private CompareTupleFilter timeComp7; |
| private CompareTupleFilter ageComp1; |
| private CompareTupleFilter ageComp2; |
| private CompareTupleFilter ageComp3; |
| private CompareTupleFilter ageComp4; |
| |
| @After |
| public void after() throws Exception { |
| |
| this.cleanupTestMetadata(); |
| } |
| |
| @Before |
| public void setup() throws IOException { |
| |
| this.createTestMetadata(); |
| |
| table = null; |
| info = table.getInfo(); |
| |
| timeComp0 = compare(info.colRef(0), FilterOperatorEnum.LT, enc(info, 0, "2015-01-14")); |
| timeComp1 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14")); |
| timeComp2 = compare(info.colRef(0), FilterOperatorEnum.LT, enc(info, 0, "2015-01-13")); |
| timeComp3 = compare(info.colRef(0), FilterOperatorEnum.LT, enc(info, 0, "2015-01-15")); |
| timeComp4 = compare(info.colRef(0), FilterOperatorEnum.EQ, enc(info, 0, "2015-01-15")); |
| timeComp5 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-15")); |
| timeComp6 = compare(info.colRef(0), FilterOperatorEnum.EQ, enc(info, 0, "2015-01-14")); |
| timeComp7 = compare(info.colRef(0), FilterOperatorEnum.EQ, enc(info, 0, "1970-01-01")); |
| ageComp1 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, "10")); |
| ageComp2 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, "20")); |
| ageComp3 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, "30")); |
| ageComp4 = compare(info.colRef(1), FilterOperatorEnum.NEQ, enc(info, 1, "30")); |
| |
| } |
| |
| @Test |
| public void verifySegmentSkipping() { |
| |
| ByteArray segmentStart = enc(info, 0, "2015-01-14"); |
| ByteArray segmentStartX = enc(info, 0, "2015-01-14 00:00:00");//when partition col is dict encoded, time format will be free |
| assertEquals(segmentStart, segmentStartX); |
| |
| { |
| LogicalTupleFilter filter = and(timeComp0, ageComp1); |
| CubeScanRangePlanner planner = new CubeScanRangePlanner(info, info.colRef(0), filter); |
| List<GTScanRange> r = planner.planScanRanges(); |
| assertEquals(1, r.size());//scan range are [close,close] |
| assertEquals("[null, 10]-[1421193600000, 10]", r.get(0).toString()); |
| assertEquals(1, r.get(0).fuzzyKeys.size()); |
| assertEquals("[[null, 10, null, null, null]]", r.get(0).fuzzyKeys.toString()); |
| } |
| { |
| LogicalTupleFilter filter = and(timeComp2, ageComp1); |
| CubeScanRangePlanner planner = new CubeScanRangePlanner(info, info.colRef(0), filter); |
| List<GTScanRange> r = planner.planScanRanges(); |
| assertEquals(1, r.size()); |
| } |
| { |
| LogicalTupleFilter filter = and(timeComp4, ageComp1); |
| CubeScanRangePlanner planner = new CubeScanRangePlanner(info, info.colRef(0), filter); |
| List<GTScanRange> r = planner.planScanRanges(); |
| assertEquals(1, r.size()); |
| } |
| { |
| LogicalTupleFilter filter = and(timeComp5, ageComp1); |
| CubeScanRangePlanner planner = new CubeScanRangePlanner(info, info.colRef(0), filter); |
| List<GTScanRange> r = planner.planScanRanges(); |
| assertEquals(1, r.size()); |
| } |
| { |
| LogicalTupleFilter filter = or(and(timeComp2, ageComp1), and(timeComp1, ageComp1), |
| and(timeComp6, ageComp1)); |
| CubeScanRangePlanner planner = new CubeScanRangePlanner(info, info.colRef(0), filter); |
| List<GTScanRange> r = planner.planScanRanges(); |
| assertEquals(2, r.size()); |
| assertEquals("[1421193600000, 10]-[null, 10]", r.get(1).toString()); |
| assertEquals("[[null, 10, null, null, null], [1421193600000, 10, null, null, null]]", |
| r.get(1).fuzzyKeys.toString()); |
| } |
| { |
| LogicalTupleFilter filter = or(and(timeComp3, ageComp3), and(timeComp7, ageComp1)); |
| CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, filter); |
| List<GTScanRange> r = planner.planScanRanges(); |
| assertEquals("[[0, 10]-[1421280000000, 30]]", r.toString()); |
| } |
| { |
| LogicalTupleFilter filter = or(timeComp2, timeComp1, timeComp6); |
| CubeScanRangePlanner planner = new CubeScanRangePlanner(info, info.colRef(0), filter); |
| List<GTScanRange> r = planner.planScanRanges(); |
| assertEquals(2, r.size()); |
| assertEquals("[1421193600000, null]-[null, null]", r.get(1).toString()); |
| assertEquals(0, r.get(1).fuzzyKeys.size()); |
| } |
| { |
| //skip FALSE filter |
| LogicalTupleFilter filter = and(ageComp1, ConstantTupleFilter.FALSE); |
| CubeScanRangePlanner planner = new CubeScanRangePlanner(info, info.colRef(0), filter); |
| List<GTScanRange> r = planner.planScanRanges(); |
| assertEquals(0, r.size()); |
| } |
| { |
| //TRUE or FALSE filter |
| LogicalTupleFilter filter = or(ConstantTupleFilter.TRUE, ConstantTupleFilter.FALSE); |
| CubeScanRangePlanner planner = new CubeScanRangePlanner(info, info.colRef(0), filter); |
| List<GTScanRange> r = planner.planScanRanges(); |
| assertEquals(1, r.size()); |
| assertEquals("[null, null]-[null, null]", r.get(0).toString()); |
| } |
| { |
| //TRUE or other filter |
| LogicalTupleFilter filter = or(ageComp1, ConstantTupleFilter.TRUE); |
| CubeScanRangePlanner planner = new CubeScanRangePlanner(info, info.colRef(0), filter); |
| List<GTScanRange> r = planner.planScanRanges(); |
| assertEquals(1, r.size()); |
| assertEquals("[null, null]-[null, null]", r.get(0).toString()); |
| } |
| } |
| |
| @Test |
| public void verifySegmentSkipping2() { |
| { |
| LogicalTupleFilter filter = and(timeComp0, ageComp1); |
| CubeScanRangePlanner planner = new CubeScanRangePlanner(info, info.colRef(0), filter); |
| List<GTScanRange> r = planner.planScanRanges(); |
| assertEquals(1, r.size());//scan range are [close,close] |
| assertEquals("[null, 10]-[1421193600000, 10]", r.get(0).toString()); |
| assertEquals(1, r.get(0).fuzzyKeys.size()); |
| assertEquals("[[null, 10, null, null, null]]", r.get(0).fuzzyKeys.toString()); |
| } |
| |
| { |
| LogicalTupleFilter filter = and(timeComp5, ageComp1); |
| CubeScanRangePlanner planner = new CubeScanRangePlanner(info, info.colRef(0), filter); |
| List<GTScanRange> r = planner.planScanRanges(); |
| assertEquals(1, r.size());//scan range are [close,close] |
| } |
| } |
| |
| @Test |
| public void verifyScanRangePlanner() { |
| |
| // flatten or-and & hbase fuzzy value |
| { |
| LogicalTupleFilter filter = and(timeComp1, or(ageComp1, ageComp2)); |
| CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, filter); |
| List<GTScanRange> r = planner.planScanRanges(); |
| assertEquals(1, r.size()); |
| assertEquals("[1421193600000, 10]-[null, 20]", r.get(0).toString()); |
| assertEquals("[[null, 10, null, null, null], [null, 20, null, null, null]]", r.get(0).fuzzyKeys.toString()); |
| } |
| |
| // pre-evaluate ever false |
| { |
| LogicalTupleFilter filter = and(timeComp1, timeComp2); |
| CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, filter); |
| List<GTScanRange> r = planner.planScanRanges(); |
| assertEquals(0, r.size()); |
| } |
| |
| // pre-evaluate ever true |
| { |
| LogicalTupleFilter filter = or(timeComp1, ageComp4); |
| CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, filter); |
| List<GTScanRange> r = planner.planScanRanges(); |
| assertEquals("[[null, null]-[null, null]]", r.toString()); |
| } |
| |
| // merge overlap range |
| { |
| LogicalTupleFilter filter = or(timeComp1, timeComp3); |
| CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, filter); |
| List<GTScanRange> r = planner.planScanRanges(); |
| assertEquals("[[null, null]-[null, null]]", r.toString()); |
| } |
| |
| // merge too many ranges |
| { |
| LogicalTupleFilter filter = or(and(timeComp4, ageComp1), and(timeComp4, ageComp2), |
| and(timeComp4, ageComp3)); |
| CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, filter); |
| List<GTScanRange> r = planner.planScanRanges(); |
| assertEquals(3, r.size()); |
| assertEquals("[1421280000000, 10]-[1421280000000, 10]", r.get(0).toString()); |
| assertEquals("[1421280000000, 20]-[1421280000000, 20]", r.get(1).toString()); |
| assertEquals("[1421280000000, 30]-[1421280000000, 30]", r.get(2).toString()); |
| planner.setMaxScanRanges(2); |
| List<GTScanRange> r2 = planner.planScanRanges(); |
| assertEquals("[[1421280000000, 10]-[1421280000000, 30]]", r2.toString()); |
| } |
| } |
| |
| @Test |
| public void verifyFirstRow() throws IOException { |
| doScanAndVerify(table, |
| new GTScanRequestBuilder().setInfo(table.getInfo()).setRanges(null).setDimensions(null) |
| .setFilterPushDown(null).createGTScanRequest(), |
| "[1421193600000, 30, Yang, 10, 10.5]", // |
| "[1421193600000, 30, Luke, 10, 10.5]", // |
| "[1421280000000, 20, Dong, 10, 10.5]", // |
| "[1421280000000, 20, Jason, 10, 10.5]", // |
| "[1421280000000, 30, Xu, 10, 10.5]", // |
| "[1421366400000, 20, Mahone, 10, 10.5]", // |
| "[1421366400000, 20, Qianhao, 10, 10.5]", // |
| "[1421366400000, 30, George, 10, 10.5]", // |
| "[1421366400000, 30, Shaofeng, 10, 10.5]", // |
| "[1421452800000, 10, Kejia, 10, 10.5]"); |
| } |
| |
| //for testing GTScanRequest serialization and deserialization |
| public static GTScanRequest useDeserializedGTScanRequest(GTScanRequest origin) { |
| ByteBuffer buffer = ByteBuffer.allocate(BytesSerializer.SERIALIZE_BUFFER_SIZE); |
| GTScanRequest.serializer.serialize(origin, buffer); |
| buffer.flip(); |
| GTScanRequest sGTScanRequest = GTScanRequest.serializer.deserialize(buffer); |
| |
| Assert.assertArrayEquals(origin.getAggrMetricsFuncs(), sGTScanRequest.getAggrMetricsFuncs()); |
| Assert.assertEquals(origin.getAggCacheMemThreshold(), sGTScanRequest.getAggCacheMemThreshold(), 0.01); |
| return sGTScanRequest; |
| } |
| |
| @Test |
| public void verifyScanWithUnevaluatableFilter() throws IOException { |
| GTInfo info = table.getInfo(); |
| |
| CompareTupleFilter fComp = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14")); |
| ExtractTupleFilter fUnevaluatable = unevaluatable(info.colRef(1)); |
| LogicalTupleFilter fNotPlusUnevaluatable = not(unevaluatable(info.colRef(1))); |
| LogicalTupleFilter filter = and(fComp, fUnevaluatable, fNotPlusUnevaluatable); |
| |
| GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null) |
| .setAggrGroupBy(setOf(0)).setAggrMetrics(setOf(3)).setAggrMetricsFuncs(new String[] { "sum" }) |
| .setFilterPushDown(filter).createGTScanRequest(); |
| |
| // note the unEvaluatable column 1 in filter is added to group by |
| assertEquals( |
| "GTScanRequest [range=[[null, null]-[null, null]], columns={0, 1, 3}, filterPushDown=AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], [], []], aggrGroupBy={0, 1}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]", |
| req.toString()); |
| |
| doScanAndVerify(table, useDeserializedGTScanRequest(req), "[1421280000000, 20, null, 20, null]", |
| "[1421280000000, 30, null, 10, null]", "[1421366400000, 20, null, 20, null]", |
| "[1421366400000, 30, null, 20, null]", "[1421452800000, 10, null, 10, null]"); |
| } |
| |
| @Test |
| public void verifyScanWithEvaluatableFilter() throws IOException { |
| GTInfo info = table.getInfo(); |
| |
| CompareTupleFilter fComp1 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14")); |
| CompareTupleFilter fComp2 = compare(info.colRef(1), FilterOperatorEnum.GT, enc(info, 1, "10")); |
| LogicalTupleFilter filter = and(fComp1, fComp2); |
| |
| GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null) |
| .setAggrGroupBy(setOf(0)).setAggrMetrics(setOf(3)).setAggrMetricsFuncs(new String[] { "sum" }) |
| .setFilterPushDown(filter).createGTScanRequest(); |
| // note the evaluatable column 1 in filter is added to returned columns but not in group by |
| assertEquals( |
| "GTScanRequest [range=[[null, null]-[null, null]], columns={0, 1, 3}, filterPushDown=AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 GT [\\x00]], aggrGroupBy={0}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]", |
| req.toString()); |
| |
| doScanAndVerify(table, useDeserializedGTScanRequest(req), "[1421280000000, 20, null, 30, null]", |
| "[1421366400000, 20, null, 40, null]"); |
| } |
| |
| @Test |
| public void verifyAggregateAndHavingFilter() throws IOException { |
| GTInfo info = table.getInfo(); |
| |
| TblColRef havingCol = TblColRef.newInnerColumn("SUM_OF_BIGDECIMAL", InnerDataTypeEnum.LITERAL); |
| havingCol.getColumnDesc().setId("1"); // point to the first aggregated measure |
| CompareTupleFilter havingFilter = compare(havingCol, FilterOperatorEnum.GT, "20"); |
| |
| GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null) |
| .setAggrGroupBy(setOf(1)).setAggrMetrics(setOf(4)).setAggrMetricsFuncs(new String[] { "sum" }) |
| .setHavingFilterPushDown(havingFilter).createGTScanRequest(); |
| |
| doScanAndVerify(table, useDeserializedGTScanRequest(req), "[null, 20, null, null, 42.0]", |
| "[null, 30, null, null, 52.5]"); |
| } |
| |
| @SuppressWarnings("unused") |
| private void testFilterScannerPerfInner(GridTable table, GTInfo info, LogicalTupleFilter filter) |
| throws IOException { |
| long start = System.currentTimeMillis(); |
| GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null) |
| .setFilterPushDown(filter).createGTScanRequest(); |
| int i = 0; |
| try (IGTScanner scanner = table.scan(req)) { |
| for (GTRecord r : scanner) { |
| i++; |
| } |
| } |
| long end = System.currentTimeMillis(); |
| System.out.println( |
| (end - start) + "ms with filter cache enabled=" + FilterResultCache.DEFAULT_OPTION + ", " + i + " rows"); |
| } |
| |
| @Test |
| public void verifyConvertFilterConstants1() { |
| GTInfo info = table.getInfo(); |
| |
| TableDesc extTable = TableDesc.mockup("ext"); |
| TblColRef extColA = TblColRef.mockup(extTable, 1, "A", "timestamp"); |
| TblColRef extColB = TblColRef.mockup(extTable, 2, "B", "integer"); |
| |
| CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14"); |
| CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.EQ, "10"); |
| LogicalTupleFilter filter = and(fComp1, fComp2); |
| |
| List<TblColRef> colMapping = Lists.newArrayList(); |
| colMapping.add(extColA); |
| colMapping.add(extColB); |
| |
| TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); |
| assertEquals( |
| "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 EQ [\\x00]]", |
| newFilter.toString()); |
| } |
| |
| @Test |
| public void verifyConvertFilterConstants2() { |
| GTInfo info = table.getInfo(); |
| |
| TableDesc extTable = TableDesc.mockup("ext"); |
| TblColRef extColA = TblColRef.mockup(extTable, 1, "A", "timestamp"); |
| TblColRef extColB = TblColRef.mockup(extTable, 2, "B", "integer"); |
| |
| List<TblColRef> colMapping = Lists.newArrayList(); |
| colMapping.add(extColA); |
| colMapping.add(extColB); |
| |
| CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14"); |
| |
| // $1<"9" round down to FALSE |
| { |
| LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.LT, "9")); |
| TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); |
| assertEquals( |
| "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 ISNOTNULL []]", |
| newFilter.toString()); |
| } |
| |
| // $1<"10" needs no rounding |
| { |
| LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.LT, "10")); |
| TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); |
| assertEquals( |
| "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 LT [\\x00]]", |
| newFilter.toString()); |
| } |
| |
| // $1<"11" round down to <="10" |
| { |
| LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.LT, "11")); |
| TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); |
| assertEquals( |
| "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 LTE [\\x00]]", |
| newFilter.toString()); |
| } |
| |
| // $1<="9" round down to FALSE |
| { |
| LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.LTE, "9")); |
| TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); |
| assertEquals(ConstantTupleFilter.FALSE, newFilter); |
| } |
| |
| // $1<="10" needs no rounding |
| { |
| LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.LTE, "10")); |
| TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); |
| assertEquals( |
| "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 LTE [\\x00]]", |
| newFilter.toString()); |
| } |
| |
| // $1<="11" round down to <="10" |
| { |
| LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.LTE, "11")); |
| TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); |
| assertEquals( |
| "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 LTE [\\x00]]", |
| newFilter.toString()); |
| } |
| } |
| |
| @Test |
| public void verifyConvertFilterConstants3() { |
| GTInfo info = table.getInfo(); |
| |
| TableDesc extTable = TableDesc.mockup("ext"); |
| TblColRef extColA = TblColRef.mockup(extTable, 1, "A", "timestamp"); |
| TblColRef extColB = TblColRef.mockup(extTable, 2, "B", "integer"); |
| |
| List<TblColRef> colMapping = Lists.newArrayList(); |
| colMapping.add(extColA); |
| colMapping.add(extColB); |
| |
| CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14"); |
| |
| // $1>"101" round up to FALSE |
| { |
| LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.GT, "101")); |
| TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); |
| assertEquals("AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 ISNOTNULL []]", |
| newFilter.toString()); |
| } |
| |
| // $1>"100" needs no rounding |
| { |
| LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.GT, "100")); |
| TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); |
| assertEquals( |
| "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 GT [\\x09]]", |
| newFilter.toString()); |
| } |
| |
| // $1>"99" round up to >="100" |
| { |
| LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.GT, "99")); |
| TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); |
| assertEquals( |
| "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 GTE [\\x09]]", |
| newFilter.toString()); |
| } |
| |
| // $1>="101" round up to FALSE |
| { |
| LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.GTE, "101")); |
| TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); |
| assertEquals(ConstantTupleFilter.FALSE, newFilter); |
| } |
| |
| // $1>="100" needs no rounding |
| { |
| LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.GTE, "100")); |
| TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); |
| assertEquals( |
| "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 GTE [\\x09]]", |
| newFilter.toString()); |
| } |
| |
| // $1>="99" round up to >="100" |
| { |
| LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.GTE, "99")); |
| TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); |
| assertEquals( |
| "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 GTE [\\x09]]", |
| newFilter.toString()); |
| } |
| } |
| |
| @Test |
| public void verifyConvertFilterConstants4() { |
| GTInfo info = table.getInfo(); |
| |
| TableDesc extTable = TableDesc.mockup("ext"); |
| TblColRef extColA = TblColRef.mockup(extTable, 1, "A", "timestamp"); |
| TblColRef extColB = TblColRef.mockup(extTable, 2, "B", "integer"); |
| |
| CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14"); |
| CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.IN, "9", "10", "15"); |
| LogicalTupleFilter filter = and(fComp1, fComp2); |
| |
| List<TblColRef> colMapping = Lists.newArrayList(); |
| colMapping.add(extColA); |
| colMapping.add(extColB); |
| |
| // $1 in ("9", "10", "15") has only "10" left |
| TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); |
| assertEquals( |
| "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 IN [\\x00]]", |
| newFilter.toString()); |
| } |
| |
| private void doScanAndVerify(GridTable table, GTScanRequest req, String... verifyRows) throws IOException { |
| System.out.println(req); |
| try (IGTScanner scanner = table.scan(req)) { |
| int i = 0; |
| for (GTRecord r : scanner) { |
| System.out.println(r); |
| if (verifyRows == null || i >= verifyRows.length) { |
| Assert.fail(); |
| } |
| assertEquals(verifyRows[i], r.toString()); |
| i++; |
| } |
| } |
| } |
| |
| public static ByteArray enc(GTInfo info, int col, String value) { |
| ByteBuffer buf = ByteBuffer.allocate(info.getMaxColumnLength()); |
| info.getCodeSystem().encodeColumnValue(col, value, buf); |
| return ByteArray.copyOf(buf.array(), buf.arrayOffset(), buf.position()); |
| } |
| |
| public static ExtractTupleFilter unevaluatable(TblColRef col) { |
| ExtractTupleFilter r = new ExtractTupleFilter(FilterOperatorEnum.EXTRACT); |
| r.addChild(new ColumnTupleFilter(col)); |
| return r; |
| } |
| |
| public static CompareTupleFilter compare(TblColRef col, FilterOperatorEnum op, Object... value) { |
| CompareTupleFilter result = new CompareTupleFilter(op); |
| result.addChild(new ColumnTupleFilter(col)); |
| result.addChild(new ConstantTupleFilter(Arrays.asList(value))); |
| return result; |
| } |
| |
| public static LogicalTupleFilter and(TupleFilter... children) { |
| return logic(FilterOperatorEnum.AND, children); |
| } |
| |
| public static LogicalTupleFilter or(TupleFilter... children) { |
| return logic(FilterOperatorEnum.OR, children); |
| } |
| |
| public static LogicalTupleFilter not(TupleFilter child) { |
| return logic(FilterOperatorEnum.NOT, child); |
| } |
| |
| public static LogicalTupleFilter logic(FilterOperatorEnum op, TupleFilter... children) { |
| LogicalTupleFilter result = new LogicalTupleFilter(op); |
| for (TupleFilter c : children) { |
| result.addChild(c); |
| } |
| return result; |
| } |
| |
| // public static GridTable newTestTable() throws IOException { |
| // GTInfo info = newInfo(); |
| // GTSimpleMemStore store = new GTSimpleMemStore(info); |
| // GridTable table = new GridTable(info, store); |
| // |
| // GTRecord r = new GTRecord(table.getInfo()); |
| // GTBuilder builder = table.rebuild(); |
| // |
| // builder.write(r.setValues("2015-01-14", "30", "Yang", new Long(10), new BigDecimal("10.5"))); |
| // builder.write(r.setValues("2015-01-14", "30", "Luke", new Long(10), new BigDecimal("10.5"))); |
| // builder.write(r.setValues("2015-01-15", "20", "Dong", new Long(10), new BigDecimal("10.5"))); |
| // builder.write(r.setValues("2015-01-15", "20", "Jason", new Long(10), new BigDecimal("10.5"))); |
| // builder.write(r.setValues("2015-01-15", "30", "Xu", new Long(10), new BigDecimal("10.5"))); |
| // builder.write(r.setValues("2015-01-16", "20", "Mahone", new Long(10), new BigDecimal("10.5"))); |
| // builder.write(r.setValues("2015-01-16", "20", "Qianhao", new Long(10), new BigDecimal("10.5"))); |
| // builder.write(r.setValues("2015-01-16", "30", "George", new Long(10), new BigDecimal("10.5"))); |
| // builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new Long(10), new BigDecimal("10.5"))); |
| // builder.write(r.setValues("2015-01-17", "10", "Kejia", new Long(10), new BigDecimal("10.5"))); |
| // builder.close(); |
| // |
| // return table; |
| // } |
| |
| // static GridTable newTestPerfTable() throws IOException { |
| // GTInfo info = newInfo(); |
| // GTSimpleMemStore store = new GTSimpleMemStore(info); |
| // GridTable table = new GridTable(info, store); |
| // |
| // GTRecord r = new GTRecord(table.getInfo()); |
| // GTBuilder builder = table.rebuild(); |
| // |
| // for (int i = 0; i < 100000; i++) { |
| // for (int j = 0; j < 10; j++) |
| // builder.write(r.setValues("2015-01-14", "30", "Yang", new Long(10), new BigDecimal("10.5"))); |
| // |
| // for (int j = 0; j < 10; j++) |
| // builder.write(r.setValues("2015-01-14", "30", "Luke", new Long(10), new BigDecimal("10.5"))); |
| // |
| // for (int j = 0; j < 10; j++) |
| // builder.write(r.setValues("2015-01-15", "20", "Dong", new Long(10), new BigDecimal("10.5"))); |
| // |
| // for (int j = 0; j < 10; j++) |
| // builder.write(r.setValues("2015-01-15", "20", "Jason", new Long(10), new BigDecimal("10.5"))); |
| // |
| // for (int j = 0; j < 10; j++) |
| // builder.write(r.setValues("2015-01-15", "30", "Xu", new Long(10), new BigDecimal("10.5"))); |
| // |
| // for (int j = 0; j < 10; j++) |
| // builder.write(r.setValues("2015-01-16", "20", "Mahone", new Long(10), new BigDecimal("10.5"))); |
| // |
| // for (int j = 0; j < 10; j++) |
| // builder.write(r.setValues("2015-01-16", "20", "Qianhao", new Long(10), new BigDecimal("10.5"))); |
| // |
| // for (int j = 0; j < 10; j++) |
| // builder.write(r.setValues("2015-01-16", "30", "George", new Long(10), new BigDecimal("10.5"))); |
| // |
| // for (int j = 0; j < 10; j++) |
| // builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new Long(10), new BigDecimal("10.5"))); |
| // |
| // for (int j = 0; j < 10; j++) |
| // builder.write(r.setValues("2015-01-17", "10", "Kejia", new Long(10), new BigDecimal("10.5"))); |
| // } |
| // builder.close(); |
| // |
| // return table; |
| // } |
| |
| // static GTInfo newInfo() { |
| // Builder builder = GTInfo.builder(); |
| // builder.setCodeSystem(newDictCodeSystem()); |
| // builder.setColumns(// |
| // DataType.getType("timestamp"), // |
| // DataType.getType("integer"), // |
| // DataType.getType("varchar(10)"), // |
| // DataType.getType("bigint"), // |
| // DataType.getType("decimal") // |
| // ); |
| // builder.setPrimaryKey(setOf(0, 1)); |
| // builder.setColumnPreferIndex(setOf(0)); |
| // builder.enableColumnBlock(new ImmutableBitSet[] { setOf(0, 1), setOf(2), setOf(3, 4) }); |
| // builder.enableRowBlock(4); |
| // GTInfo info = builder.build(); |
| // return info; |
| // } |
| |
| // private static CubeCodeSystem newDictCodeSystem() { |
| // DimensionEncoding[] dimEncs = new DimensionEncoding[3]; |
| // dimEncs[1] = new DictionaryDimEnc(newDictionaryOfInteger()); |
| // dimEncs[2] = new DictionaryDimEnc(newDictionaryOfString()); |
| // return new CubeCodeSystem(dimEncs); |
| // } |
| |
| // private static Dictionary newDictionaryOfString() { |
| // TrieDictionaryBuilder<String> builder = new TrieDictionaryBuilder<>(new StringBytesConverter()); |
| // builder.addValue("Dong"); |
| // builder.addValue("George"); |
| // builder.addValue("Jason"); |
| // builder.addValue("Kejia"); |
| // builder.addValue("Luke"); |
| // builder.addValue("Mahone"); |
| // builder.addValue("Qianhao"); |
| // builder.addValue("Shaofeng"); |
| // builder.addValue("Xu"); |
| // builder.addValue("Yang"); |
| // return builder.build(0); |
| // } |
| // |
| // private static Dictionary newDictionaryOfInteger() { |
| // NumberDictionaryForestBuilder builder = new NumberDictionaryForestBuilder(); |
| // builder.addValue("10"); |
| // builder.addValue("20"); |
| // builder.addValue("30"); |
| // builder.addValue("40"); |
| // builder.addValue("50"); |
| // builder.addValue("60"); |
| // builder.addValue("70"); |
| // builder.addValue("80"); |
| // builder.addValue("90"); |
| // builder.addValue("100"); |
| // return builder.build(); |
| // } |
| |
| public static ImmutableBitSet setOf(int... values) { |
| BitSet set = new BitSet(); |
| for (int i : values) |
| set.set(i); |
| return new ImmutableBitSet(set); |
| } |
| } |
