| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| #include "orc/BloomFilter.hh" |
| #include "orc/Common.hh" |
| #include "orc/Type.hh" |
| #include "PredicateLeaf.hh" |
| |
| #include <algorithm> |
| #include <functional> |
| #include <sstream> |
| #include <type_traits> |
| |
| namespace orc { |
| |
| PredicateLeaf::PredicateLeaf(Operator op, |
| PredicateDataType type, |
| const std::string& colName, |
| Literal literal) |
| : mOperator(op) |
| , mType(type) |
| , mColumnName(colName) |
| , mHasColumnName(true) |
| , mColumnId(0) { |
| mLiterals.emplace_back(literal); |
| mHashCode = hashCode(); |
| validate(); |
| } |
| |
| PredicateLeaf::PredicateLeaf(Operator op, |
| PredicateDataType type, |
| uint64_t columnId, |
| Literal literal) |
| : mOperator(op) |
| , mType(type) |
| , mHasColumnName(false) |
| , mColumnId(columnId) { |
| mLiterals.emplace_back(literal); |
| mHashCode = hashCode(); |
| validate(); |
| } |
| |
| PredicateLeaf::PredicateLeaf(Operator op, |
| PredicateDataType type, |
| const std::string& colName, |
| const std::initializer_list<Literal>& literals) |
| : mOperator(op) |
| , mType(type) |
| , mColumnName(colName) |
| , mHasColumnName(true) |
| , mLiterals(literals.begin(), literals.end()) { |
| mHashCode = hashCode(); |
| validate(); |
| } |
| |
| PredicateLeaf::PredicateLeaf(Operator op, |
| PredicateDataType type, |
| uint64_t columnId, |
| const std::initializer_list<Literal>& literals) |
| : mOperator(op) |
| , mType(type) |
| , mHasColumnName(false) |
| , mColumnId(columnId) |
| , mLiterals(literals.begin(), literals.end()) { |
| mHashCode = hashCode(); |
| validate(); |
| } |
| |
| PredicateLeaf::PredicateLeaf(Operator op, |
| PredicateDataType type, |
| const std::string& colName, |
| const std::vector<Literal>& literals) |
| : mOperator(op) |
| , mType(type) |
| , mColumnName(colName) |
| , mHasColumnName(true) |
| , mLiterals(literals.begin(), literals.end()) { |
| mHashCode = hashCode(); |
| validate(); |
| } |
| |
| PredicateLeaf::PredicateLeaf(Operator op, |
| PredicateDataType type, |
| uint64_t columnId, |
| const std::vector<Literal>& literals) |
| : mOperator(op) |
| , mType(type) |
| , mHasColumnName(false) |
| , mColumnId(columnId) |
| , mLiterals(literals.begin(), literals.end()) { |
| mHashCode = hashCode(); |
| validate(); |
| } |
| |
| void PredicateLeaf::validateColumn() const { |
| if (mHasColumnName && mColumnName.empty()) { |
| throw std::invalid_argument("column name should not be empty"); |
| } else if (!mHasColumnName && mColumnId == INVALID_COLUMN_ID) { |
| throw std::invalid_argument("invalid column id"); |
| } |
| } |
| |
| void PredicateLeaf::validate() const { |
| switch (mOperator) { |
| case Operator::IS_NULL: |
| validateColumn(); |
| if (!mLiterals.empty()) { |
| throw std::invalid_argument("No literal is required!"); |
| } |
| break; |
| case Operator::EQUALS: |
| case Operator::NULL_SAFE_EQUALS: |
| case Operator::LESS_THAN: |
| case Operator::LESS_THAN_EQUALS: |
| validateColumn(); |
| if (mLiterals.size() != 1) { |
| throw std::invalid_argument("One literal is required!"); |
| } |
| if (static_cast<int>(mLiterals.at(0).getType()) != |
| static_cast<int>(mType)) { |
| throw std::invalid_argument("leaf and literal types do not match!"); |
| } |
| break; |
| case Operator::IN: |
| validateColumn(); |
| if (mLiterals.size() < 2) { |
| throw std::invalid_argument("At least two literals are required!"); |
| } |
| for (auto literal : mLiterals) { |
| if (static_cast<int>(literal.getType()) != static_cast<int>(mType)) { |
| throw std::invalid_argument("leaf and literal types do not match!"); |
| } |
| } |
| break; |
| case Operator::BETWEEN: |
| validateColumn(); |
| for (auto literal : mLiterals) { |
| if (static_cast<int>(literal.getType()) != static_cast<int>(mType)) { |
| throw std::invalid_argument("leaf and literal types do not match!"); |
| } |
| } |
| break; |
| default: |
| break; |
| } |
| } |
| |
| PredicateLeaf::Operator PredicateLeaf::getOperator() const { |
| return mOperator; |
| } |
| |
| PredicateDataType PredicateLeaf::getType() const { |
| return mType; |
| } |
| |
| bool PredicateLeaf::hasColumnName() const { |
| return mHasColumnName; |
| } |
| |
| /** |
| * Get the simple column name. |
| */ |
| const std::string& PredicateLeaf::getColumnName() const { |
| return mColumnName; |
| } |
| |
| uint64_t PredicateLeaf::getColumnId() const { |
| return mColumnId; |
| } |
| |
| /** |
| * Get the literal half of the predicate leaf. |
| */ |
| Literal PredicateLeaf::getLiteral() const { |
| return mLiterals.at(0); |
| } |
| |
| /** |
| * For operators with multiple literals (IN and BETWEEN), get the literals. |
| */ |
| const std::vector<Literal>& PredicateLeaf::getLiteralList() const { |
| return mLiterals; |
| } |
| |
| static std::string getLiteralString(const std::vector<Literal>& literals) { |
| return literals.at(0).toString(); |
| } |
| |
| static std::string getLiteralsString(const std::vector<Literal>& literals) { |
| std::ostringstream sstream; |
| sstream << "["; |
| for (size_t i = 0; i != literals.size(); ++i) { |
| sstream << literals[i].toString(); |
| if (i + 1 != literals.size()) { |
| sstream << ", "; |
| } |
| } |
| sstream << "]"; |
| return sstream.str(); |
| } |
| |
| std::string PredicateLeaf::columnDebugString() const { |
| if (mHasColumnName) return mColumnName; |
| std::ostringstream sstream; |
| sstream << "column(id=" << mColumnId << ')'; |
| return sstream.str(); |
| } |
| |
| std::string PredicateLeaf::toString() const { |
| std::ostringstream sstream; |
| sstream << '('; |
| switch (mOperator) { |
| case Operator::IS_NULL: |
| sstream << columnDebugString() << " is null"; |
| break; |
| case Operator::EQUALS: |
| sstream << columnDebugString() << " = " << getLiteralString(mLiterals); |
| break; |
| case Operator::NULL_SAFE_EQUALS: |
| sstream << columnDebugString() << " null_safe_= " |
| << getLiteralString(mLiterals); |
| break; |
| case Operator::LESS_THAN: |
| sstream << columnDebugString() << " < " << getLiteralString(mLiterals); |
| break; |
| case Operator::LESS_THAN_EQUALS: |
| sstream << columnDebugString() << " <= " << getLiteralString(mLiterals); |
| break; |
| case Operator::IN: |
| sstream << columnDebugString() << " in " << getLiteralsString(mLiterals); |
| break; |
| case Operator::BETWEEN: |
| sstream << columnDebugString() << " between " << getLiteralsString(mLiterals); |
| break; |
| default: |
| sstream << "unknown operator, column: " |
| << columnDebugString() << ", literals: " |
| << getLiteralsString(mLiterals); |
| } |
| sstream << ')'; |
| return sstream.str(); |
| } |
| |
| size_t PredicateLeaf::hashCode() const { |
| size_t value = 0; |
| std::for_each(mLiterals.cbegin(), mLiterals.cend(), |
| [&](const Literal& literal) { |
| value = value * 17 + literal.getHashCode(); |
| }); |
| auto colHash = mHasColumnName ? |
| std::hash<std::string>{}(mColumnName) : |
| std::hash<uint64_t>{}(mColumnId); |
| return value * 103 * 101 * 3 * 17 + |
| std::hash<int>{}(static_cast<int>(mOperator)) + |
| std::hash<int>{}(static_cast<int>(mType)) * 17 + |
| colHash * 3 * 17; |
| } |
| |
| bool PredicateLeaf::operator==(const PredicateLeaf& r) const { |
| if (this == &r) { |
| return true; |
| } |
| if (mHashCode != r.mHashCode || mType != r.mType || mOperator != r.mOperator || |
| mHasColumnName != r.mHasColumnName || mColumnName != r.mColumnName || |
| mColumnId != r.mColumnId || mLiterals.size() != r.mLiterals.size()) { |
| return false; |
| } |
| for (size_t i = 0; i != mLiterals.size(); ++i) { |
| if (mLiterals[i] != r.mLiterals[i]) { |
| return false; |
| } |
| } |
| return true; |
| } |
| |
| // enum to mark the position of predicate in the range |
| enum class Location { |
| BEFORE, MIN, MIDDLE, MAX, AFTER |
| }; |
| |
| DIAGNOSTIC_PUSH |
| DIAGNOSTIC_IGNORE("-Wfloat-equal") |
| |
| /** |
| * Given a point and min and max, determine if the point is before, at the |
| * min, in the middle, at the max, or after the range. |
| * @param point the point to test |
| * @param min the minimum point |
| * @param max the maximum point |
| * @return the location of the point |
| */ |
| template <typename T> |
| Location compareToRange(const T& point, const T& min, const T& max) { |
| if (point < min) { |
| return Location::BEFORE; |
| } else if (point == min) { |
| return Location::MIN; |
| } |
| |
| if (point > max) { |
| return Location::AFTER; |
| } else if (point == max) { |
| return Location::MAX; |
| } |
| |
| return Location::MIDDLE; |
| } |
| |
| /** |
| * Evaluate a predicate leaf according to min/max values |
| * @param op operator of the predicate |
| * @param values the value to test |
| * @param minValue the minimum value |
| * @param maxValue the maximum value |
| * @param hasNull whether the statistics contain null |
| * @return the TruthValue result of the test |
| */ |
| template <typename T> |
| TruthValue evaluatePredicateRange(const PredicateLeaf::Operator op, |
| const std::vector<T>& values, |
| const T& minValue, |
| const T& maxValue, |
| bool hasNull) { |
| Location loc; |
| switch (op) { |
| case PredicateLeaf::Operator::NULL_SAFE_EQUALS: |
| loc = compareToRange(values.at(0), minValue, maxValue); |
| if (loc == Location::BEFORE || loc == Location::AFTER) { |
| return TruthValue::NO; |
| } else { |
| return TruthValue::YES_NO; |
| } |
| case PredicateLeaf::Operator::EQUALS: |
| loc = compareToRange(values.at(0), minValue, maxValue); |
| if (minValue == maxValue && loc == Location::MIN) { |
| return hasNull ? TruthValue::YES_NULL : TruthValue::YES; |
| } else if (loc == Location::BEFORE || loc == Location::AFTER) { |
| return hasNull ? TruthValue::NO_NULL : TruthValue::NO; |
| } else { |
| return hasNull ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; |
| } |
| case PredicateLeaf::Operator::LESS_THAN: |
| loc = compareToRange(values.at(0), minValue, maxValue); |
| if (loc == Location::AFTER) { |
| return hasNull ? TruthValue::YES_NULL : TruthValue::YES; |
| } else if (loc == Location::BEFORE || loc == Location::MIN) { |
| return hasNull ? TruthValue::NO_NULL : TruthValue::NO; |
| } else { |
| return hasNull ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; |
| } |
| case PredicateLeaf::Operator::LESS_THAN_EQUALS: |
| loc = compareToRange(values.at(0), minValue, maxValue); |
| if (loc == Location::AFTER || loc == Location::MAX || |
| (loc == Location::MIN && minValue == maxValue)) { |
| return hasNull ? TruthValue::YES_NULL : TruthValue::YES; |
| } else if (loc == Location::BEFORE) { |
| return hasNull ? TruthValue::NO_NULL : TruthValue::NO; |
| } else { |
| return hasNull ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; |
| } |
| case PredicateLeaf::Operator::IN: |
| if (minValue == maxValue) { |
| // for a single value, look through to see if that value is in the set |
| for (auto& value : values) { |
| loc = compareToRange(value, minValue, maxValue); |
| if (loc == Location::MIN) { |
| return hasNull ? TruthValue::YES_NULL : TruthValue::YES; |
| } |
| } |
| return hasNull ? TruthValue::NO_NULL : TruthValue::NO; |
| } else { |
| // are all of the values outside of the range? |
| for (auto& value : values) { |
| loc = compareToRange(value, minValue, maxValue); |
| if (loc == Location::MIN || loc == Location::MIDDLE || |
| loc == Location::MAX) { |
| return hasNull ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; |
| } |
| } |
| return hasNull ? TruthValue::NO_NULL : TruthValue::NO; |
| } |
| case PredicateLeaf::Operator::BETWEEN: |
| if (values.empty()) { |
| return TruthValue::YES_NO; |
| } |
| loc = compareToRange(values.at(0), minValue, maxValue); |
| if (loc == Location::BEFORE || loc == Location::MIN) { |
| Location loc2 = compareToRange(values.at(1), minValue, maxValue); |
| if (loc2 == Location::AFTER || loc2 == Location::MAX) { |
| return hasNull ? TruthValue::YES_NULL : TruthValue::YES; |
| } else if (loc2 == Location::BEFORE) { |
| return hasNull ? TruthValue::NO_NULL : TruthValue::NO; |
| } else { |
| return hasNull ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; |
| } |
| } else if (loc == Location::AFTER) { |
| return hasNull ? TruthValue::NO_NULL : TruthValue::NO; |
| } else { |
| return hasNull ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; |
| } |
| case PredicateLeaf::Operator::IS_NULL: |
| // min = null condition above handles the all-nulls YES case |
| return hasNull ? TruthValue::YES_NO : TruthValue::NO; |
| default: |
| return hasNull ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; |
| } |
| } |
| |
| DIAGNOSTIC_POP |
| |
| static TruthValue evaluateBoolPredicate( |
| const PredicateLeaf::Operator op, |
| const std::vector<Literal>& literals, |
| const proto::ColumnStatistics& stats) { |
| bool hasNull = stats.hasnull(); |
| if (!stats.has_bucketstatistics() || |
| stats.bucketstatistics().count_size() == 0) { |
| // does not have bool stats |
| return hasNull ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; |
| } |
| |
| auto trueCount = stats.bucketstatistics().count(0); |
| auto falseCount = stats.numberofvalues() - trueCount; |
| switch (op) { |
| case PredicateLeaf::Operator::IS_NULL: |
| return hasNull ? TruthValue::YES_NO : TruthValue::NO; |
| case PredicateLeaf::Operator::NULL_SAFE_EQUALS: { |
| if (literals.at(0).getBool()) { |
| if (trueCount == 0) { |
| return TruthValue::NO; |
| } else if (falseCount == 0) { |
| return TruthValue::YES; |
| } |
| } else { |
| if (falseCount == 0) { |
| return TruthValue::NO; |
| } else if (trueCount == 0) { |
| return TruthValue::YES; |
| } |
| } |
| return TruthValue::YES_NO; |
| } |
| case PredicateLeaf::Operator::EQUALS: { |
| if (literals.at(0).getBool()) { |
| if (trueCount == 0) { |
| return hasNull ? TruthValue::NO_NULL : TruthValue::NO; |
| } else if (falseCount == 0) { |
| return hasNull ? TruthValue::YES_NULL : TruthValue::YES; |
| } |
| } else { |
| if (falseCount == 0) { |
| return hasNull ? TruthValue::NO_NULL : TruthValue::NO; |
| } else if (trueCount == 0) { |
| return hasNull ? TruthValue::YES_NULL : TruthValue::YES; |
| } |
| } |
| return hasNull ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; |
| } |
| case PredicateLeaf::Operator::LESS_THAN: |
| case PredicateLeaf::Operator::LESS_THAN_EQUALS: |
| case PredicateLeaf::Operator::IN: |
| case PredicateLeaf::Operator::BETWEEN: |
| default: |
| return hasNull ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; |
| } |
| } |
| |
| static std::vector<int64_t> literal2Long(const std::vector<Literal>& values) { |
| std::vector<int64_t> result; |
| std::for_each(values.cbegin(), values.cend(), [&](const Literal& val) { |
| if (!val.isNull()) { |
| result.emplace_back(val.getLong()); |
| } |
| }); |
| return result; |
| } |
| |
| static std::vector<int32_t> literal2Date(const std::vector<Literal>& values) { |
| std::vector<int32_t> result; |
| std::for_each(values.cbegin(), values.cend(), [&](const Literal& val) { |
| if (!val.isNull()) { |
| result.emplace_back(val.getDate()); |
| } |
| }); |
| return result; |
| } |
| |
| static std::vector<Literal::Timestamp> literal2Timestamp( |
| const std::vector<Literal>& values) { |
| std::vector<Literal::Timestamp> result; |
| std::for_each(values.cbegin(), values.cend(), [&](const Literal& val) { |
| if (!val.isNull()) { |
| result.emplace_back(val.getTimestamp()); |
| } |
| }); |
| return result; |
| } |
| |
| static std::vector<Decimal> literal2Decimal( |
| const std::vector<Literal>& values) { |
| std::vector<Decimal> result; |
| std::for_each(values.cbegin(), values.cend(), [&](const Literal& val) { |
| if (!val.isNull()) { |
| result.emplace_back(val.getDecimal()); |
| } |
| }); |
| return result; |
| } |
| |
| static std::vector<double> literal2Double( |
| const std::vector<Literal>& values) { |
| std::vector<double> result; |
| std::for_each(values.cbegin(), values.cend(), [&](const Literal& val) { |
| if (!val.isNull()) { |
| result.emplace_back(val.getFloat()); |
| } |
| }); |
| return result; |
| } |
| |
| static std::vector<std::string> literal2String( |
| const std::vector<Literal>& values) { |
| std::vector<std::string> result; |
| std::for_each(values.cbegin(), values.cend(), [&](const Literal& val) { |
| if (!val.isNull()) { |
| result.emplace_back(val.getString()); |
| } |
| }); |
| return result; |
| } |
| |
| TruthValue PredicateLeaf::evaluatePredicateMinMax( |
| const proto::ColumnStatistics& colStats) const { |
| TruthValue result = TruthValue::YES_NO_NULL; |
| switch (mType) { |
| case PredicateDataType::LONG: { |
| if (colStats.has_intstatistics() && |
| colStats.intstatistics().has_minimum() && |
| colStats.intstatistics().has_maximum()) { |
| const auto& stats = colStats.intstatistics(); |
| result = evaluatePredicateRange( |
| mOperator, |
| literal2Long(mLiterals), |
| stats.minimum(), |
| stats.maximum(), |
| colStats.hasnull()); |
| } |
| break; |
| } |
| case PredicateDataType::FLOAT: { |
| if (colStats.has_doublestatistics() && |
| colStats.doublestatistics().has_minimum() && |
| colStats.doublestatistics().has_maximum()) { |
| const auto& stats = colStats.doublestatistics(); |
| if (!std::isfinite(stats.sum())) { |
| result = colStats.hasnull() ? |
| TruthValue::YES_NO_NULL : TruthValue::YES_NO; |
| } else { |
| result = evaluatePredicateRange( |
| mOperator, |
| literal2Double(mLiterals), |
| stats.minimum(), |
| stats.maximum(), |
| colStats.hasnull()); |
| } |
| } |
| break; |
| } |
| case PredicateDataType::STRING: { |
| ///TODO: check lowerBound and upperBound as well |
| if (colStats.has_stringstatistics() && |
| colStats.stringstatistics().has_minimum() && |
| colStats.stringstatistics().has_maximum()) { |
| const auto& stats = colStats.stringstatistics(); |
| result = evaluatePredicateRange( |
| mOperator, |
| literal2String(mLiterals), |
| stats.minimum(), |
| stats.maximum(), |
| colStats.hasnull()); |
| } |
| break; |
| } |
| case PredicateDataType::DATE: { |
| if (colStats.has_datestatistics() && |
| colStats.datestatistics().has_minimum() && |
| colStats.datestatistics().has_maximum()) { |
| const auto& stats = colStats.datestatistics(); |
| result = evaluatePredicateRange( |
| mOperator, |
| literal2Date(mLiterals), |
| stats.minimum(), |
| stats.maximum(), |
| colStats.hasnull()); |
| } |
| break; |
| } |
| case PredicateDataType::TIMESTAMP: { |
| if (colStats.has_timestampstatistics() && |
| colStats.timestampstatistics().has_minimumutc() && |
| colStats.timestampstatistics().has_maximumutc()) { |
| const auto& stats = colStats.timestampstatistics(); |
| constexpr int32_t DEFAULT_MIN_NANOS = 0; |
| constexpr int32_t DEFAULT_MAX_NANOS = 999999; |
| int32_t minNano = stats.has_minimumnanos() ? |
| stats.minimumnanos() - 1 : DEFAULT_MIN_NANOS; |
| int32_t maxNano = stats.has_maximumnanos() ? |
| stats.maximumnanos() - 1 : DEFAULT_MAX_NANOS; |
| Literal::Timestamp minTimestamp( |
| stats.minimumutc() / 1000, |
| static_cast<int32_t>((stats.minimumutc() % 1000) * 1000000) + minNano); |
| Literal::Timestamp maxTimestamp( |
| stats.maximumutc() / 1000, |
| static_cast<int32_t>((stats.maximumutc() % 1000) * 1000000) + maxNano); |
| result = evaluatePredicateRange( |
| mOperator, |
| literal2Timestamp(mLiterals), |
| minTimestamp, |
| maxTimestamp, |
| colStats.hasnull()); |
| } |
| break; |
| } |
| case PredicateDataType::DECIMAL: { |
| if (colStats.has_decimalstatistics() && |
| colStats.decimalstatistics().has_minimum() && |
| colStats.decimalstatistics().has_maximum()) { |
| const auto& stats = colStats.decimalstatistics(); |
| result = evaluatePredicateRange( |
| mOperator, |
| literal2Decimal(mLiterals), |
| Decimal(stats.minimum()), |
| Decimal(stats.maximum()), |
| colStats.hasnull()); |
| } |
| break; |
| } |
| case PredicateDataType::BOOLEAN: { |
| if (colStats.has_bucketstatistics()) { |
| result = evaluateBoolPredicate(mOperator, mLiterals, colStats); |
| } |
| break; |
| } |
| default: |
| break; |
| } |
| |
| // make sure null literal is respected for IN operator |
| if (mOperator == Operator::IN && colStats.hasnull()) { |
| for (const auto& literal : mLiterals) { |
| if (literal.isNull()) { |
| result = TruthValue::YES_NO_NULL; |
| break; |
| } |
| } |
| } |
| |
| return result; |
| } |
| |
| static bool shouldEvaluateBloomFilter(PredicateLeaf::Operator op, |
| TruthValue result, |
| const BloomFilter * bloomFilter) { |
| // evaluate bloom filter only when |
| // 1) Bloom filter is available |
| // 2) Min/Max evaluation yield YES or MAYBE |
| // 3) Predicate is EQUALS or IN list |
| // 4) Decimal type stores its string representation |
| // but has inconsistency in trailing zeros |
| if (bloomFilter != nullptr |
| && result != TruthValue::NO_NULL && result != TruthValue::NO |
| && (op == PredicateLeaf::Operator::EQUALS |
| || op == PredicateLeaf::Operator::NULL_SAFE_EQUALS |
| || op == PredicateLeaf::Operator::IN)) { |
| return true; |
| } |
| return false; |
| } |
| |
| static TruthValue checkInBloomFilter(PredicateLeaf::Operator, |
| PredicateDataType type, |
| const Literal& literal, |
| const BloomFilter * bf, |
| bool hasNull) { |
| TruthValue result = hasNull ? TruthValue::NO_NULL : TruthValue::NO; |
| if (literal.isNull()) { |
| result = hasNull ? TruthValue::YES_NO_NULL : TruthValue::NO; |
| } else if (type == PredicateDataType::LONG) { |
| if (bf->testLong(literal.getLong())) { |
| result = TruthValue::YES_NO_NULL; |
| } |
| } else if (type == PredicateDataType::FLOAT) { |
| if (bf->testDouble(literal.getFloat())) { |
| result = TruthValue::YES_NO_NULL; |
| } |
| } else if (type == PredicateDataType::STRING) { |
| std::string str = literal.getString(); |
| if (bf->testBytes(str.c_str(), static_cast<int64_t>(str.size()))) { |
| result = TruthValue::YES_NO_NULL; |
| } |
| } else if (type == PredicateDataType::DECIMAL) { |
| std::string decimal = literal.getDecimal().toString(true); |
| if (bf->testBytes(decimal.c_str(), static_cast<int64_t>(decimal.size()))) { |
| result = TruthValue::YES_NO_NULL; |
| } |
| } else if (type == PredicateDataType::TIMESTAMP) { |
| if (bf->testLong(literal.getTimestamp().getMillis())) { |
| result = TruthValue::YES_NO_NULL; |
| } |
| } else if (type == PredicateDataType::DATE) { |
| if (bf->testLong(literal.getDate())) { |
| result = TruthValue::YES_NO_NULL; |
| } |
| } else { |
| result = TruthValue::YES_NO_NULL; |
| } |
| |
| if (result == TruthValue::YES_NO_NULL && !hasNull) { |
| result = TruthValue::YES_NO; |
| } |
| |
| return result; |
| } |
| |
| TruthValue PredicateLeaf::evaluatePredicateBloomFiter(const BloomFilter * bf, |
| bool hasNull) const { |
| switch (mOperator) { |
| case Operator::NULL_SAFE_EQUALS: |
| // null safe equals does not return *_NULL variant. |
| // So set hasNull to false |
| return checkInBloomFilter( |
| mOperator, mType, mLiterals.front(), bf, false); |
| case Operator::EQUALS: |
| return checkInBloomFilter( |
| mOperator, mType, mLiterals.front(), bf, hasNull); |
| case Operator::IN: |
| for (const auto &literal : mLiterals) { |
| // if at least one value in IN list exist in bloom filter, |
| // qualify the row group/stripe |
| TruthValue result = checkInBloomFilter( |
| mOperator, mType, literal, bf, hasNull); |
| if (result == TruthValue::YES_NO_NULL || |
| result == TruthValue::YES_NO) { |
| return result; |
| } |
| } |
| return hasNull ? TruthValue::NO_NULL : TruthValue::NO; |
| case Operator::LESS_THAN: |
| case Operator::LESS_THAN_EQUALS: |
| case Operator::BETWEEN: |
| case Operator::IS_NULL: |
| default: |
| return hasNull ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; |
| } |
| } |
| |
| TruthValue PredicateLeaf::evaluate(const WriterVersion writerVersion, |
| const proto::ColumnStatistics& colStats, |
| const BloomFilter * bloomFilter) const { |
| // files written before ORC-135 stores timestamp wrt to local timezone |
| // causing issues with PPD. disable PPD for timestamp for all old files |
| if (mType == PredicateDataType::TIMESTAMP) { |
| if (writerVersion < WriterVersion::WriterVersion_ORC_135) { |
| return TruthValue::YES_NO_NULL; |
| } |
| } |
| |
| bool allNull = colStats.hasnull() && colStats.numberofvalues() == 0; |
| if (mOperator == Operator::IS_NULL || (( |
| mOperator == Operator::EQUALS || |
| mOperator == Operator::NULL_SAFE_EQUALS) && |
| mLiterals.at(0).isNull())) { |
| // IS_NULL operator does not need to check min/max stats and bloom filter |
| return allNull ? TruthValue::YES : |
| (colStats.hasnull() ? TruthValue::YES_NO : TruthValue::NO); |
| } else if (allNull) { |
| // if we don't have any value, everything must have been null |
| return TruthValue::IS_NULL; |
| } |
| |
| TruthValue result = evaluatePredicateMinMax(colStats); |
| if (shouldEvaluateBloomFilter(mOperator, result, bloomFilter)) { |
| return evaluatePredicateBloomFiter(bloomFilter, colStats.hasnull()); |
| } else { |
| return result; |
| } |
| } |
| |
| } // namespace orc |
