core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.carbondata.core.scan.filter;

import java.util.ArrayList;
import java.util.List;
import java.util.UUID;

import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.LiteralExpression;
import org.apache.carbondata.core.scan.expression.conditional.InExpression;
import org.apache.carbondata.core.scan.expression.conditional.ListExpression;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.expression.logical.AndExpression;
import org.apache.carbondata.core.scan.expression.logical.TrueExpression;
import org.apache.carbondata.core.scan.filter.intf.RowImpl;
import org.apache.carbondata.core.util.BitSetGroup;

import mockit.Mock;
import mockit.MockUp;
import org.junit.Before;
import org.junit.Test;

import static junit.framework.TestCase.assertFalse;
import static junit.framework.TestCase.assertTrue;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;

public class FilterUtilTest {

  private ColumnSchema columnSchema;

  @Before public void setUp() throws Exception {
    columnSchema = new ColumnSchema();
    columnSchema.setColumnName("IMEI");
    columnSchema.setColumnUniqueId(UUID.randomUUID().toString());
    columnSchema.setDataType(DataTypes.STRING);
    columnSchema.setDimensionColumn(true);
  }

  @Test public void testCheckIfLeftExpressionRequireEvaluation() {
    List<Expression> children = new ArrayList<>();
    ListExpression expression = new ListExpression(children);
    boolean result = FilterUtil.checkIfLeftExpressionRequireEvaluation(expression);
    assertTrue(result);
  }

  @Test
  public void testCheckIfLeftExpressionRequireEvaluationWithExpressionNotInstanceOfColumnExpression() {
    ColumnExpression expression = new ColumnExpression("test", DataTypes.STRING);
    boolean result = FilterUtil.checkIfLeftExpressionRequireEvaluation(expression);
    assertFalse(result);
  }

  @Test public void testNanSafeEqualsDoublesWithUnEqualValues() {
    Double d1 = new Double(60.67);
    Double d2 = new Double(60.69);
    boolean result = FilterUtil.nanSafeEqualsDoubles(d1, d2);
    assertFalse(result);
  }

  @Test public void testNanSafeEqualsDoublesWithEqualValues() {
    Double d1 = new Double(60.67);
    Double d2 = new Double(60.67);
    boolean result = FilterUtil.nanSafeEqualsDoubles(d1, d2);
    assertTrue(result);
  }

  @Test public void testCompareFilterKeyBasedOnDataTypeForShortValue() {
    String dictionaryVal = "1";
    String memberVal = "1";
    int actualResult =
        FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataTypes.SHORT);
    int expectedResult = 0;
    assertEquals(expectedResult, actualResult);
  }

  @Test public void testCompareFilterKeyBasedOnDataTypeForIntValue() {
    String dictionaryVal = "1000";
    String memberVal = "1001";
    int actualResult =
        FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataTypes.INT);
    int expectedResult = -1;
    assertEquals(expectedResult, actualResult);
  }

  @Test public void testCompareFilterKeyBasedOnDataTypeForDoubleValue() {
    String dictionaryVal = "1.90";
    String memberVal = "1.89";
    int actualResult =
        FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataTypes.DOUBLE);
    int expectedResult = 1;
    assertEquals(expectedResult, actualResult);
  }

  @Test public void testCompareFilterKeyBasedOnDataTypeForLongValue() {
    String dictionaryVal = "111111111111111";
    String memberVal = "1111111111111111";
    int actualResult =
        FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataTypes.LONG);
    int expectedResult = -1;
    assertEquals(expectedResult, actualResult);
  }

  @Test public void testCompareFilterKeyBasedOnDataTypeForBooleanValue() {
    String dictionaryVal = "true";
    String memberVal = "false";
    int actualResult =
        FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataTypes.BOOLEAN);
    int expectedResult = 1;
    assertEquals(expectedResult, actualResult);
  }

  @Test public void testCompareFilterKeyBasedOnDataTypeForDecimalValue() {
    String dictionaryVal = "1111111";
    String memberVal = "1111";
    int actualResult =
        FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataTypes.createDefaultDecimalType());
    int expectedResult = 1;
    assertEquals(expectedResult, actualResult);
  }

  @Test public void testCompareFilterKeyBasedOnDataTypeForDefaultValue() {
    String dictionaryVal = "11.78";
    String memberVal = "1111.90";
    int actualResult =
        FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataTypes.FLOAT);
    int expectedResult = -1;
    assertEquals(expectedResult, actualResult);
  }

  @Test public void testCompareFilterKeyBasedOnDataTypeForTimestamp() {
    String dictionaryVal = "2008-01-01 00:00:01";
    String memberVal = "2008-01-01 00:00:01";
    int actualValue =
        FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataTypes.TIMESTAMP);
    int expectedValue = 0;
    assertEquals(expectedValue, actualValue);
  }

  @Test public void testCompareFilterKeyBasedOnDataTypeForException() throws Exception {
    String dictionaryVal = "test";
    String memberVal = "1";
    int actualValue =
        FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataTypes.INT);
    int expectedValue = -1;
    assertEquals(expectedValue, actualValue);
  }

  @Test public void testCreateIndexKeyFromResolvedFilterVal() throws Exception {
    long[] startOrEndKey = new long[] { 0, 10 };
    byte[] startOrEndKeyForNoDictDimension = { 1, 2 };
    int[] keys = new int[] { 1, 2 };
    MultiDimKeyVarLengthGenerator multiDimKeyVarLengthGenerator =
        new MultiDimKeyVarLengthGenerator(keys);
    assertTrue(FilterUtil
        .createIndexKeyFromResolvedFilterVal(startOrEndKey, multiDimKeyVarLengthGenerator,
            startOrEndKeyForNoDictDimension) != null);

  }

  @Test public void testCheckIfExpressionContainsColumn() {
    String columnName = "IMEI";
    Expression expression = new ColumnExpression(columnName, DataTypes.STRING);
    boolean result = FilterUtil.checkIfExpressionContainsColumn(expression);
    assertTrue(result);
  }

  @Test
  public void testCheckIfExpressionContainsColumnWithExpressionNotInstanceOfColumnExpression() {
    String columnName = "IMEI";
    Expression expression = new LiteralExpression(columnName, DataTypes.STRING);
    boolean result = FilterUtil.checkIfExpressionContainsColumn(expression);
    assertFalse(result);
  }

  @Test public void testIsExpressionNeedsToResolved() {
    boolean isIncludeFilter = true;
    Object obj = "test";
    LiteralExpression literalExpression = new LiteralExpression(obj, DataTypes.STRING);
    boolean result = FilterUtil.isExpressionNeedsToResolved(literalExpression, isIncludeFilter);
    assertFalse(result);
  }

  @Test public void testIsExpressionNeedsToResolvedWithDataTypeNullAndIsIncludeFilterFalse() {
    boolean isIncludeFilter = false;
    Object obj = "test";
    LiteralExpression literalExpression = new LiteralExpression(obj, DataTypes.NULL);
    boolean result = FilterUtil.isExpressionNeedsToResolved(literalExpression, isIncludeFilter);
    assertTrue(result);
  }

  @Test public void testGetMaskKey() {
    int surrogate = 1;
    int[] keys = new int[] { 1, 2 };
    MultiDimKeyVarLengthGenerator multiDimKeyVarLengthGenerator =
        new MultiDimKeyVarLengthGenerator(keys);
    int ordinal = 1;
    int keyOrdinal = 1;
    int complexTypeOrdinal = 1;
    ColumnSchema columnSchema = new ColumnSchema();
    columnSchema.setColumnName("IMEI");
    columnSchema.setColumnUniqueId(UUID.randomUUID().toString());
    columnSchema.setDataType(DataTypes.STRING);
    columnSchema.setDimensionColumn(true);
    CarbonDimension carbonDimension =
        new CarbonDimension(columnSchema, ordinal, keyOrdinal, complexTypeOrdinal);
    byte[] expectedResult = new byte[] { 1 };
    byte[] actualResult =
        FilterUtil.getMaskKey(surrogate, carbonDimension, multiDimKeyVarLengthGenerator);
    assertArrayEquals(expectedResult, actualResult);
  }

  @Test public void testGetFilterListForAllMembersRS() throws Exception {
    Expression expression = new ColumnExpression("IMEI", DataTypes.STRING);
    ColumnExpression columnExpression = new ColumnExpression("IMEI", DataTypes.STRING);
    String defaultValues = "test";
    int defaultSurrogate = 1;
    boolean isIncludeFilter = true;
    int ordinal = 1;
    ColumnSchema dimColumn = new ColumnSchema();
    dimColumn.setColumnName("IMEI");
    dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
    dimColumn.setDataType(DataTypes.STRING);
    dimColumn.setDimensionColumn(true);
    final CarbonColumn carbonColumn = new CarbonColumn(dimColumn, ordinal, -1);
    new MockUp<ColumnExpression>() {
      @Mock public CarbonColumn getCarbonColumn() {
        return carbonColumn;
      }
    };

    new MockUp<RowImpl>() {
      @Mock public Object getVal(int index) {
        return "test";
      }
    };
    assertTrue(FilterUtil
        .getFilterListForAllMembersRS(expression, columnExpression, defaultValues, defaultSurrogate,
            isIncludeFilter) instanceof ColumnFilterInfo);
  }

  @Test public void testGetFilterListForAllMembersRSWithDefaultValuesEqualsToNull()
      throws Exception {
    Expression expression = new ColumnExpression("IMEI", DataTypes.STRING);
    ColumnExpression columnExpression = new ColumnExpression("IMEI", DataTypes.STRING);
    String defaultValues = CarbonCommonConstants.MEMBER_DEFAULT_VAL;
    int defaultSurrogate = 1;
    boolean isIncludeFilter = true;
    int ordinal = 1;
    ColumnSchema dimColumn = new ColumnSchema();
    dimColumn.setColumnName("IMEI");
    dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
    dimColumn.setDataType(DataTypes.STRING);
    dimColumn.setDimensionColumn(true);
    final CarbonColumn carbonColumn = new CarbonColumn(dimColumn, ordinal, -1);
    new MockUp<ColumnExpression>() {
      @Mock public CarbonColumn getCarbonColumn() {
        return carbonColumn;
      }
    };

    new MockUp<RowImpl>() {
      @Mock public Object getVal(int index) {
        return "test";
      }
    };
    assertTrue(FilterUtil
        .getFilterListForAllMembersRS(expression, columnExpression, defaultValues, defaultSurrogate,
            isIncludeFilter) instanceof ColumnFilterInfo);
  }

  @Test public void testgetFilterListForRS() throws Exception {
    Expression expression = new ColumnExpression("IMEI", DataTypes.STRING);
    ColumnExpression columnExpression = new ColumnExpression("IMEI", DataTypes.STRING);
    String defaultValues = CarbonCommonConstants.MEMBER_DEFAULT_VAL;
    int defaultSurrogate = 1;
    int ordinal = 1;
    final CarbonColumn carbonColumn = new CarbonColumn(columnSchema, ordinal, -1);
    new MockUp<ColumnExpression>() {
      @Mock public CarbonColumn getCarbonColumn() {
        return carbonColumn;
      }
    };

    new MockUp<RowImpl>() {
      @Mock public Object getVal(int index) {
        return "test";
      }
    };
    assertTrue(FilterUtil.getFilterListForRS(expression, defaultValues,
        defaultSurrogate) instanceof ColumnFilterInfo);
  }

  @Test public void testCheckIfDataTypeNotTimeStamp() {
    Expression expression = new ColumnExpression("test", DataTypes.STRING);
    boolean result = FilterUtil.checkIfDataTypeNotTimeStamp(expression);
    assertFalse(result);
  }

  @Test public void testPrepareDefaultEndIndexKey() throws Exception {
    List<ColumnSchema> columnsInTable = new ArrayList<>();
    columnsInTable.add(columnSchema);
    int[] columnCardinality = new int[] { 1, 2 };
    new MockUp<ColumnSchema>() {
      @Mock public List<Encoding> getEncodingList() {
        List<Encoding> encodingList = new ArrayList<>();
        encodingList.add(Encoding.DICTIONARY);
        return encodingList;
      }
    };
    SegmentProperties segmentProperties = new SegmentProperties(columnsInTable, columnCardinality);
    assertTrue(FilterUtil.prepareDefaultEndIndexKey(segmentProperties) instanceof IndexKey);
  }

  @Test public void testCheckIfRightExpressionRequireEvaluation() {
    Expression expression = new ColumnExpression("test", DataTypes.STRING);
    boolean result = FilterUtil.checkIfRightExpressionRequireEvaluation(expression);
    assertTrue(result);
  }

  @Test
  public void testCheckIfRightExpressionRequireEvaluationWithExpressionIsInstanceOfLiteralExpression() {
    Expression expression = new LiteralExpression("test", DataTypes.STRING);
    boolean result = FilterUtil.checkIfRightExpressionRequireEvaluation(expression);
    assertFalse(result);
  }

  @Test public void testGetNoDictionaryValKeyMemberForFilter() throws FilterUnsupportedException {
    boolean isIncludeFilter = true;
    ColumnExpression expression = new ColumnExpression("test", DataTypes.STRING);
    List<String> evaluateResultListFinal = new ArrayList<>();
    evaluateResultListFinal.add("test1");
    evaluateResultListFinal.add("test2");
    assertTrue(FilterUtil
        .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, isIncludeFilter,
            DataTypes.STRING) instanceof ColumnFilterInfo);
  }

  @Test public void testPrepareDefaultStartIndexKey() throws KeyGenException {
    List<ColumnSchema> columnsInTable = new ArrayList<>();
    columnsInTable.add(columnSchema);
    int[] columnCardinality = new int[] { 1, 2 };
    new MockUp<ColumnSchema>() {
      @Mock public List<Encoding> getEncodingList() {
        List<Encoding> encodingList = new ArrayList<>();
        encodingList.add(Encoding.DICTIONARY);
        return encodingList;
      }
    };
    SegmentProperties segmentProperties = new SegmentProperties(columnsInTable, columnCardinality);
    assertTrue(FilterUtil.prepareDefaultStartIndexKey(segmentProperties) instanceof IndexKey);
  }

  @Test public void testCreateBitSetGroupWithDefaultValue() {
    // test for exactly divisible values
    BitSetGroup bitSetGroupWithDefaultValue =
        FilterUtil.createBitSetGroupWithDefaultValue(14, 448000, true);
    assertTrue(bitSetGroupWithDefaultValue.getNumberOfPages() == 14);
    // test for remainder values
    bitSetGroupWithDefaultValue =
        FilterUtil.createBitSetGroupWithDefaultValue(15, 448200, true);
    assertTrue(bitSetGroupWithDefaultValue.getNumberOfPages() == 15);
  }

  @Test public void testRemoveInExpressionNodeWithPositionIdColumn() {
    List<Expression> children = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    // create literal expression
    LiteralExpression literalExpression =
        new LiteralExpression("0/1/0-0_batchno0-0-1517808273200/0", DataTypes.STRING);
    children.add(literalExpression);
    // create list expression
    ListExpression listExpression = new ListExpression(children);
    // create column expression with column name as positionId
    ColumnExpression columnExpression =
        new ColumnExpression(CarbonCommonConstants.POSITION_ID, DataTypes.STRING);
    // create InExpression as right node
    InExpression inExpression = new InExpression(columnExpression, listExpression);
    // create a dummy true expression as left node
    TrueExpression trueExpression = new TrueExpression(null);
    // create and expression as the root node
    Expression expression = new AndExpression(trueExpression, inExpression);
    // test remove expression method
    FilterUtil.removeInExpressionNodeWithPositionIdColumn(expression);
    // after removing the right node instance of right node should be of true expression
    assert (((AndExpression) expression).getRight() instanceof TrueExpression);
  }

  @Test public void testRemoveInExpressionNodeWithDifferentColumn() {
    List<Expression> children = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    // create literal expression
    LiteralExpression literalExpression =
        new LiteralExpression("testName", DataTypes.STRING);
    children.add(literalExpression);
    // create list expression
    ListExpression listExpression = new ListExpression(children);
    // create column expression with column name as positionId
    ColumnExpression columnExpression = new ColumnExpression("name", DataTypes.STRING);
    // create InExpression as right node
    InExpression inExpression = new InExpression(columnExpression, listExpression);
    // create a dummy true expression as left node
    TrueExpression trueExpression = new TrueExpression(null);
    // create and expression as the root node
    Expression expression = new AndExpression(trueExpression, inExpression);
    // test remove expression method
    FilterUtil.removeInExpressionNodeWithPositionIdColumn(expression);
    // after removing the right node instance of right node should be of true expression
    assert (((AndExpression) expression).getRight() instanceof InExpression);
  }
}