| /* | |
| * Licensed to the Apache Software Foundation (ASF) under one | |
| * or more contributor license agreements. See the NOTICE file | |
| * distributed with this work for additional information | |
| * regarding copyright ownership. The ASF licenses this file | |
| * to you under the Apache License, Version 2.0 (the | |
| * "License"); you may not use this file except in compliance | |
| * with the License. You may obtain a copy of the License at | |
| * | |
| * http://www.apache.org/licenses/LICENSE-2.0 | |
| * | |
| * Unless required by applicable law or agreed to in writing, software | |
| * distributed under the License is distributed on an "AS IS" BASIS, | |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| * See the License for the specific language governing permissions and | |
| * limitations under the License. | |
| */ | |
| #include "paimon/common/predicate/literal_converter.h" | |
| #include <algorithm> | |
| #include <cmath> | |
| #include <cstddef> | |
| #include <variant> | |
| #include "arrow/api.h" | |
| #include "arrow/array/array_base.h" | |
| #include "arrow/array/array_dict.h" | |
| #include "arrow/ipc/json_simple.h" | |
| #include "gtest/gtest.h" | |
| #include "paimon/common/data/binary_row.h" | |
| #include "paimon/common/data/data_define.h" | |
| #include "paimon/common/utils/decimal_utils.h" | |
| #include "paimon/data/decimal.h" | |
| #include "paimon/data/timestamp.h" | |
| #include "paimon/defs.h" | |
| #include "paimon/memory/memory_pool.h" | |
| #include "paimon/status.h" | |
| #include "paimon/testing/utils/binary_row_generator.h" | |
| #include "paimon/testing/utils/testharness.h" | |
| namespace paimon::test { | |
| class LiteralConverterTest : public ::testing::Test { | |
| public: | |
| void SetUp() override {} | |
| void TearDown() override {} | |
| void CheckResult(const std::shared_ptr<arrow::Array>& field_array, | |
| const std::vector<Literal>& expected) const { | |
| ASSERT_OK_AND_ASSIGN( | |
| std::vector<Literal> literals, | |
| LiteralConverter::ConvertLiteralsFromArray(*field_array, /*own_data=*/false)); | |
| ASSERT_EQ(literals.size(), expected.size()); | |
| ASSERT_EQ(literals, expected); | |
| } | |
| void CheckLiteralsFromString(const FieldType& type, const std::vector<std::string>& strs, | |
| const std::vector<Literal>& expected) const { | |
| ASSERT_EQ(strs.size(), expected.size()); | |
| for (size_t i = 0; i < strs.size(); i++) { | |
| ASSERT_OK_AND_ASSIGN(auto result, | |
| LiteralConverter::ConvertLiteralsFromString(type, strs[i])); | |
| ASSERT_EQ(result, expected[i]); | |
| } | |
| } | |
| void CheckLiteralFromRow(const std::shared_ptr<arrow::DataType>& data_type, | |
| const BinaryRowGenerator::ValueType& values, const FieldType& type, | |
| const std::vector<Literal>& expected) const { | |
| auto pool = GetDefaultPool(); | |
| auto schema = arrow::schema(arrow::FieldVector({arrow::field("f0", data_type)})); | |
| for (size_t i = 0; i < values.size(); i++) { | |
| // each value generates a row with 1 arity | |
| BinaryRow row = BinaryRowGenerator::GenerateRow({values[i]}, pool.get()); | |
| ASSERT_OK_AND_ASSIGN(auto result, LiteralConverter::ConvertLiteralsFromRow( | |
| schema, row, /*field_idx=*/0, type)); | |
| ASSERT_EQ(result, expected[i]); | |
| } | |
| } | |
| void CheckLiteralsToArray(const FieldType& type, const std::vector<Literal>& literals, | |
| const std::shared_ptr<arrow::Array>& expected) const { | |
| ASSERT_OK_AND_ASSIGN( | |
| std::shared_ptr<arrow::Array> array, | |
| LiteralConverter::ConvertLiteralsToArray(type, literals, arrow::default_memory_pool())); | |
| ASSERT_TRUE(array->Equals(*expected)) | |
| << "actual: " << array->ToString() << ", expected: " << expected->ToString(); | |
| // the array reads back as the literals it was built from, the null ones included | |
| CheckResult(array, literals); | |
| } | |
| }; | |
| TEST_F(LiteralConverterTest, TestBooleanLiteral) { | |
| auto field_array = | |
| arrow::ipc::internal::json::ArrayFromJSON(arrow::boolean(), R"([true, false, null])") | |
| .ValueOrDie(); | |
| CheckResult(field_array, | |
| std::vector<Literal>({Literal(true), Literal(false), Literal(FieldType::BOOLEAN)})); | |
| CheckLiteralsFromString( | |
| FieldType::BOOLEAN, {"true", "false", "yes", "no"}, | |
| std::vector<Literal>({Literal(true), Literal(false), Literal(true), Literal(false)})); | |
| CheckLiteralFromRow(arrow::boolean(), {true, false, NullType()}, FieldType::BOOLEAN, | |
| {Literal(true), Literal(false), Literal(FieldType::BOOLEAN)}); | |
| } | |
| TEST_F(LiteralConverterTest, TestTinyIntLiteral) { | |
| auto field_array = | |
| arrow::ipc::internal::json::ArrayFromJSON(arrow::int8(), R"([4, 5, null])").ValueOrDie(); | |
| CheckResult(field_array, std::vector<Literal>({Literal(static_cast<int8_t>(4)), | |
| Literal(static_cast<int8_t>(5)), | |
| Literal(FieldType::TINYINT)})); | |
| CheckLiteralsFromString( | |
| FieldType::TINYINT, {"4", "5"}, | |
| std::vector<Literal>({Literal(static_cast<int8_t>(4)), Literal(static_cast<int8_t>(5))})); | |
| CheckLiteralFromRow(arrow::int8(), {static_cast<int8_t>(4), static_cast<int8_t>(5), NullType()}, | |
| FieldType::TINYINT, | |
| {Literal(static_cast<int8_t>(4)), Literal(static_cast<int8_t>(5)), | |
| Literal(FieldType::TINYINT)}); | |
| } | |
| TEST_F(LiteralConverterTest, TestSmallIntLiteral) { | |
| auto field_array = | |
| arrow::ipc::internal::json::ArrayFromJSON(arrow::int16(), R"([45, 55, null])").ValueOrDie(); | |
| CheckResult(field_array, std::vector<Literal>({Literal(static_cast<int16_t>(45)), | |
| Literal(static_cast<int16_t>(55)), | |
| Literal(FieldType::SMALLINT)})); | |
| CheckLiteralsFromString(FieldType::SMALLINT, {"45", "55"}, | |
| std::vector<Literal>({Literal(static_cast<int16_t>(45)), | |
| Literal(static_cast<int16_t>(55))})); | |
| CheckLiteralFromRow(arrow::int16(), | |
| {static_cast<int16_t>(45), static_cast<int16_t>(55), NullType()}, | |
| FieldType::SMALLINT, | |
| {Literal(static_cast<int16_t>(45)), Literal(static_cast<int16_t>(55)), | |
| Literal(FieldType::SMALLINT)}); | |
| } | |
| TEST_F(LiteralConverterTest, TestIntLiteral) { | |
| auto field_array = | |
| arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), R"([456, 567, null])") | |
| .ValueOrDie(); | |
| CheckResult(field_array, | |
| std::vector<Literal>({Literal(456), Literal(567), Literal(FieldType::INT)})); | |
| CheckLiteralsFromString(FieldType::INT, {"456", "567"}, | |
| std::vector<Literal>({Literal(456), Literal(567)})); | |
| CheckLiteralFromRow(arrow::int32(), | |
| {static_cast<int32_t>(456), static_cast<int32_t>(567), NullType()}, | |
| FieldType::INT, | |
| {Literal(static_cast<int32_t>(456)), Literal(static_cast<int32_t>(567)), | |
| Literal(FieldType::INT)}); | |
| } | |
| TEST_F(LiteralConverterTest, TestBigIntLiteral) { | |
| auto field_array = | |
| arrow::ipc::internal::json::ArrayFromJSON(arrow::int64(), R"([4, 5, null])").ValueOrDie(); | |
| CheckResult(field_array, | |
| std::vector<Literal>({Literal(4l), Literal(5l), Literal(FieldType::BIGINT)})); | |
| CheckLiteralsFromString(FieldType::BIGINT, {"4", "5"}, | |
| std::vector<Literal>({Literal(4l), Literal(5l)})); | |
| CheckLiteralFromRow(arrow::int64(), | |
| {static_cast<int64_t>(4), static_cast<int64_t>(5), NullType()}, | |
| FieldType::BIGINT, | |
| {Literal(static_cast<int64_t>(4)), Literal(static_cast<int64_t>(5)), | |
| Literal(FieldType::BIGINT)}); | |
| } | |
| TEST_F(LiteralConverterTest, TestFloatLiteral) { | |
| auto field_array = | |
| arrow::ipc::internal::json::ArrayFromJSON(arrow::float32(), R"([4.0, 5.1, NaN, null])") | |
| .ValueOrDie(); | |
| CheckResult(field_array, | |
| std::vector<Literal>( | |
| {Literal(static_cast<float>(4.0)), Literal(static_cast<float>(5.1)), | |
| Literal(static_cast<float>(std::nan(""))), Literal(FieldType::FLOAT)})); | |
| // literal from string do not support nan and inf | |
| CheckLiteralsFromString(FieldType::FLOAT, {"4.0", "5.1"}, | |
| std::vector<Literal>({Literal(4.0f), Literal(5.1f)})); | |
| CheckLiteralFromRow(arrow::float32(), {4.0f, 5.1f, INFINITY, -INFINITY, NAN, NullType()}, | |
| FieldType::FLOAT, | |
| {Literal(4.0f), Literal(5.1f), Literal(INFINITY), Literal(-INFINITY), | |
| Literal(NAN), Literal(FieldType::FLOAT)}); | |
| } | |
| TEST_F(LiteralConverterTest, TestDoubleLiteral) { | |
| auto field_array = | |
| arrow::ipc::internal::json::ArrayFromJSON(arrow::float64(), R"([4.05, 5.17, NaN, null])") | |
| .ValueOrDie(); | |
| CheckResult(field_array, | |
| std::vector<Literal>({Literal(4.05), Literal(5.17), Literal(std::nan("")), | |
| Literal(FieldType::DOUBLE)})); | |
| // literal from string do not support nan and inf | |
| CheckLiteralsFromString(FieldType::DOUBLE, {"4.05", "5.17"}, | |
| std::vector<Literal>({Literal(4.05), Literal(5.17)})); | |
| CheckLiteralFromRow(arrow::float64(), | |
| {4.05, 5.17, static_cast<double> INFINITY, static_cast<double>(-INFINITY), | |
| static_cast<double> NAN, NullType()}, | |
| FieldType::DOUBLE, | |
| {Literal(4.05), Literal(5.17), Literal(static_cast<double> INFINITY), | |
| Literal(static_cast<double>(-INFINITY)), Literal(static_cast<double> NAN), | |
| Literal(FieldType::DOUBLE)}); | |
| } | |
| TEST_F(LiteralConverterTest, TestStringLiteral) { | |
| auto field_array = | |
| arrow::ipc::internal::json::ArrayFromJSON(arrow::utf8(), R"(["apple", "苹果", null])") | |
| .ValueOrDie(); | |
| std::string str = "苹果"; | |
| CheckResult(field_array, | |
| std::vector<Literal>({Literal(FieldType::STRING, "apple", 5), | |
| Literal(FieldType::STRING, str.data(), str.size()), | |
| Literal(FieldType::STRING)})); | |
| CheckLiteralsFromString( | |
| FieldType::STRING, {"apple", "苹果"}, | |
| std::vector<Literal>({Literal(FieldType::STRING, "apple", 5), | |
| Literal(FieldType::STRING, str.data(), str.size())})); | |
| CheckLiteralFromRow( | |
| arrow::utf8(), {std::string("apple"), std::string("苹果"), NullType()}, FieldType::STRING, | |
| {Literal(FieldType::STRING, "apple", 5), Literal(FieldType::STRING, str.data(), str.size()), | |
| Literal(FieldType::STRING)}); | |
| } | |
| TEST_F(LiteralConverterTest, TestBinaryLiteral) { | |
| auto field_array = | |
| arrow::ipc::internal::json::ArrayFromJSON(arrow::binary(), R"(["apple", "苹果", null])") | |
| .ValueOrDie(); | |
| std::string str = "苹果"; | |
| CheckResult(field_array, | |
| std::vector<Literal>({Literal(FieldType::BINARY, "apple", 5), | |
| Literal(FieldType::BINARY, str.data(), str.size()), | |
| Literal(FieldType::BINARY)})); | |
| CheckLiteralsFromString( | |
| FieldType::BINARY, {"apple", "苹果"}, | |
| std::vector<Literal>({Literal(FieldType::BINARY, "apple", 5), | |
| Literal(FieldType::BINARY, str.data(), str.size())})); | |
| CheckLiteralFromRow( | |
| arrow::binary(), {std::string("apple"), std::string("苹果"), NullType()}, FieldType::BINARY, | |
| {Literal(FieldType::BINARY, "apple", 5), Literal(FieldType::BINARY, str.data(), str.size()), | |
| Literal(FieldType::BINARY)}); | |
| } | |
| TEST_F(LiteralConverterTest, TestTimestampLiteral) { | |
| { | |
| // nano | |
| std::string timestamp_json = | |
| R"(["1970-01-01T00:00:59.123456789", "2000-02-29T23:23:23.999999999", | |
| "1899-01-01T00:59:20.001001001", "2033-05-18T03:33:20.000000000", | |
| "2020-01-01T01:05:05.001", "2010-01-03T06:30:30.006163", | |
| "2010-01-04T07:35:35", "2008-12-28", "2012-01-01 01:02:03", null])"; | |
| auto field_array = arrow::ipc::internal::json::ArrayFromJSON( | |
| arrow::timestamp(arrow::TimeUnit::NANO), timestamp_json) | |
| .ValueOrDie(); | |
| CheckResult( | |
| field_array, | |
| std::vector<Literal>( | |
| {Literal(Timestamp(59123l, 456789)), Literal(Timestamp(951866603999l, 999999)), | |
| Literal(Timestamp(-2240521239999l, 1001)), Literal(Timestamp(2000000000000l, 0)), | |
| Literal(Timestamp(1577840705001l, 0)), Literal(Timestamp(1262500230006l, 163000)), | |
| Literal(Timestamp(1262590535000l, 0)), Literal(Timestamp(1230422400000l, 0)), | |
| Literal(Timestamp(1325379723000l, 0)), Literal(FieldType::TIMESTAMP)})); | |
| CheckLiteralFromRow( | |
| arrow::timestamp(arrow::TimeUnit::NANO), | |
| {TimestampType(Timestamp(59123l, 456789), 9), | |
| TimestampType(Timestamp(951866603999l, 999999), 9), | |
| TimestampType(Timestamp(-2240521239999l, 1001), 9), | |
| TimestampType(Timestamp(2000000000000l, 0), 9), | |
| TimestampType(Timestamp(1577840705001l, 0), 9), | |
| TimestampType(Timestamp(1262500230006l, 163000), 9), | |
| TimestampType(Timestamp(1262590535000l, 0), 9), | |
| TimestampType(Timestamp(1230422400000l, 0), 9), | |
| TimestampType(Timestamp(1325379723000l, 0), 9), NullType()}, | |
| FieldType::TIMESTAMP, | |
| {Literal(Timestamp(59123l, 456789)), Literal(Timestamp(951866603999l, 999999)), | |
| Literal(Timestamp(-2240521239999l, 1001)), Literal(Timestamp(2000000000000l, 0)), | |
| Literal(Timestamp(1577840705001l, 0)), Literal(Timestamp(1262500230006l, 163000)), | |
| Literal(Timestamp(1262590535000l, 0)), Literal(Timestamp(1230422400000l, 0)), | |
| Literal(Timestamp(1325379723000l, 0)), Literal(FieldType::TIMESTAMP)}); | |
| } | |
| { | |
| // second | |
| std::string timestamp_json = | |
| R"(["1970-01-01T00:00:59", "2000-02-29T23:23:23", | |
| "1899-01-01T00:59:20", "2033-05-18T03:33:20", | |
| "2020-01-01T01:05:05", "2010-01-03T06:30:30", | |
| "2010-01-04T07:35:35", "2008-12-28", "2012-01-01 01:02:03", null])"; | |
| auto field_array = arrow::ipc::internal::json::ArrayFromJSON( | |
| arrow::timestamp(arrow::TimeUnit::SECOND), timestamp_json) | |
| .ValueOrDie(); | |
| CheckResult( | |
| field_array, | |
| std::vector<Literal>( | |
| {Literal(Timestamp(59000l, 0)), Literal(Timestamp(951866603000l, 0)), | |
| Literal(Timestamp(-2240521240000l, 0)), Literal(Timestamp(2000000000000l, 0)), | |
| Literal(Timestamp(1577840705000l, 0)), Literal(Timestamp(1262500230000l, 0)), | |
| Literal(Timestamp(1262590535000l, 0)), Literal(Timestamp(1230422400000l, 0)), | |
| Literal(Timestamp(1325379723000l, 0)), Literal(FieldType::TIMESTAMP)})); | |
| CheckLiteralFromRow( | |
| arrow::timestamp(arrow::TimeUnit::SECOND), | |
| {TimestampType(Timestamp(59000l, 0), 0), TimestampType(Timestamp(951866603000l, 0), 0), | |
| TimestampType(Timestamp(-2240521240000l, 0), 0), | |
| TimestampType(Timestamp(2000000000000l, 0), 0), | |
| TimestampType(Timestamp(1577840705000l, 0), 0), | |
| TimestampType(Timestamp(1262500230000l, 0), 0), | |
| TimestampType(Timestamp(1262590535000l, 0), 0), | |
| TimestampType(Timestamp(1230422400000l, 0), 0), | |
| TimestampType(Timestamp(1325379723000l, 0), 0), NullType()}, | |
| FieldType::TIMESTAMP, | |
| {Literal(Timestamp(59000l, 0)), Literal(Timestamp(951866603000l, 0)), | |
| Literal(Timestamp(-2240521240000l, 0)), Literal(Timestamp(2000000000000l, 0)), | |
| Literal(Timestamp(1577840705000l, 0)), Literal(Timestamp(1262500230000l, 0)), | |
| Literal(Timestamp(1262590535000l, 0)), Literal(Timestamp(1230422400000l, 0)), | |
| Literal(Timestamp(1325379723000l, 0)), Literal(FieldType::TIMESTAMP)}); | |
| } | |
| { | |
| // milli | |
| std::string timestamp_json = | |
| R"(["1970-01-01T00:00:59.001", "2000-02-29T23:23:23.001", | |
| "1899-01-01T00:59:20.001", "2033-05-18T03:33:20.001", | |
| "2020-01-01T01:05:05.001", "2010-01-03T06:30:30.001", | |
| "2010-01-04T07:35:35.001", "2008-12-28", "2012-01-01 01:02:03.001", null])"; | |
| auto field_array = arrow::ipc::internal::json::ArrayFromJSON( | |
| arrow::timestamp(arrow::TimeUnit::MILLI), timestamp_json) | |
| .ValueOrDie(); | |
| CheckResult( | |
| field_array, | |
| std::vector<Literal>( | |
| {Literal(Timestamp(59001l, 0)), Literal(Timestamp(951866603001l, 0)), | |
| Literal(Timestamp(-2240521239999l, 0)), Literal(Timestamp(2000000000001l, 0)), | |
| Literal(Timestamp(1577840705001l, 0)), Literal(Timestamp(1262500230001l, 0)), | |
| Literal(Timestamp(1262590535001l, 0)), Literal(Timestamp(1230422400000l, 0)), | |
| Literal(Timestamp(1325379723001l, 0)), Literal(FieldType::TIMESTAMP)})); | |
| CheckLiteralFromRow( | |
| arrow::timestamp(arrow::TimeUnit::MILLI), | |
| {TimestampType(Timestamp(59001l, 0), 3), TimestampType(Timestamp(951866603001l, 0), 3), | |
| TimestampType(Timestamp(-2240521239999l, 0), 3), | |
| TimestampType(Timestamp(2000000000001l, 0), 3), | |
| TimestampType(Timestamp(1577840705001l, 0), 3), | |
| TimestampType(Timestamp(1262500230001l, 0), 3), | |
| TimestampType(Timestamp(1262590535001l, 0), 3), | |
| TimestampType(Timestamp(1230422400000l, 0), 3), | |
| TimestampType(Timestamp(1325379723001l, 0), 3), NullType()}, | |
| FieldType::TIMESTAMP, | |
| {Literal(Timestamp(59001l, 0)), Literal(Timestamp(951866603001l, 0)), | |
| Literal(Timestamp(-2240521239999l, 0)), Literal(Timestamp(2000000000001l, 0)), | |
| Literal(Timestamp(1577840705001l, 0)), Literal(Timestamp(1262500230001l, 0)), | |
| Literal(Timestamp(1262590535001l, 0)), Literal(Timestamp(1230422400000l, 0)), | |
| Literal(Timestamp(1325379723001l, 0)), Literal(FieldType::TIMESTAMP)}); | |
| } | |
| { | |
| // micro | |
| std::string timestamp_json = | |
| R"(["1970-01-01T00:00:59.001001", "2000-02-29T23:23:23.001001", | |
| "1899-01-01T00:59:20.001001", "2033-05-18T03:33:20.001001", | |
| "2020-01-01T01:05:05.001001", "2010-01-03T06:30:30.001001", | |
| "2010-01-04T07:35:35.001001", "2008-12-28", "2012-01-01 01:02:03.001001", null])"; | |
| auto field_array = arrow::ipc::internal::json::ArrayFromJSON( | |
| arrow::timestamp(arrow::TimeUnit::MICRO), timestamp_json) | |
| .ValueOrDie(); | |
| CheckResult( | |
| field_array, | |
| std::vector<Literal>( | |
| {Literal(Timestamp(59001l, 1000)), Literal(Timestamp(951866603001l, 1000)), | |
| Literal(Timestamp(-2240521239999l, 1000)), | |
| Literal(Timestamp(2000000000001l, 1000)), Literal(Timestamp(1577840705001l, 1000)), | |
| Literal(Timestamp(1262500230001l, 1000)), Literal(Timestamp(1262590535001l, 1000)), | |
| Literal(Timestamp(1230422400000l, 0)), Literal(Timestamp(1325379723001l, 1000)), | |
| Literal(FieldType::TIMESTAMP)})); | |
| CheckLiteralFromRow( | |
| arrow::timestamp(arrow::TimeUnit::MICRO), | |
| {TimestampType(Timestamp(59001l, 1000), 6), | |
| TimestampType(Timestamp(951866603001l, 1000), 6), | |
| TimestampType(Timestamp(-2240521239999l, 1000), 6), | |
| TimestampType(Timestamp(2000000000001l, 1000), 6), | |
| TimestampType(Timestamp(1577840705001l, 1000), 6), | |
| TimestampType(Timestamp(1262500230001l, 1000), 6), | |
| TimestampType(Timestamp(1262590535001l, 1000), 6), | |
| TimestampType(Timestamp(1230422400000l, 0), 6), | |
| TimestampType(Timestamp(1325379723001l, 1000), 6), NullType()}, | |
| FieldType::TIMESTAMP, | |
| {Literal(Timestamp(59001l, 1000)), Literal(Timestamp(951866603001l, 1000)), | |
| Literal(Timestamp(-2240521239999l, 1000)), Literal(Timestamp(2000000000001l, 1000)), | |
| Literal(Timestamp(1577840705001l, 1000)), Literal(Timestamp(1262500230001l, 1000)), | |
| Literal(Timestamp(1262590535001l, 1000)), Literal(Timestamp(1230422400000l, 0)), | |
| Literal(Timestamp(1325379723001l, 1000)), Literal(FieldType::TIMESTAMP)}); | |
| } | |
| } | |
| TEST_F(LiteralConverterTest, TestDecimalLiteral) { | |
| auto field_array = | |
| arrow::ipc::internal::json::ArrayFromJSON( | |
| arrow::decimal128(21, 3), | |
| R"(["-123456789987654321.234", "123456789987654321.012", "0.000", "123.456", "-123.456", null])") | |
| .ValueOrDie(); | |
| CheckResult( | |
| field_array, | |
| std::vector<Literal>( | |
| {Literal(Decimal(21, 3, DecimalUtils::StrToInt128("-123456789987654321234").value())), | |
| Literal(Decimal(21, 3, DecimalUtils::StrToInt128("123456789987654321012").value())), | |
| Literal(Decimal(21, 3, 0)), Literal(Decimal(21, 3, 123456)), | |
| Literal(Decimal(21, 3, -123456)), Literal(FieldType::DECIMAL)})); | |
| CheckLiteralFromRow( | |
| arrow::decimal128(38, 3), | |
| {Decimal(38, 3, DecimalUtils::StrToInt128("-123456789987654338234").value()), | |
| Decimal(38, 3, DecimalUtils::StrToInt128("123456789987654338012").value()), | |
| Decimal(38, 3, 0), Decimal(38, 3, 123456), Decimal(38, 3, -123456), NullType()}, | |
| FieldType::DECIMAL, | |
| {Literal(Decimal(38, 3, DecimalUtils::StrToInt128("-123456789987654338234").value())), | |
| Literal(Decimal(38, 3, DecimalUtils::StrToInt128("123456789987654338012").value())), | |
| Literal(Decimal(38, 3, 0)), Literal(Decimal(38, 3, 123456)), | |
| Literal(Decimal(38, 3, -123456)), Literal(FieldType::DECIMAL)}); | |
| } | |
| TEST_F(LiteralConverterTest, TestDateLiteral) { | |
| auto field_array = | |
| arrow::ipc::internal::json::ArrayFromJSON(arrow::date32(), R"([0, 4, -5, null])") | |
| .ValueOrDie(); | |
| CheckResult(field_array, | |
| std::vector<Literal>({Literal(FieldType::DATE, 0l), Literal(FieldType::DATE, 4l), | |
| Literal(FieldType::DATE, -5l), Literal(FieldType::DATE)})); | |
| CheckLiteralsFromString( | |
| FieldType::DATE, {"1", "0", "1970-01-02", "1969-12-31"}, | |
| std::vector<Literal>({Literal(FieldType::DATE, 1), Literal(FieldType::DATE, 0), | |
| Literal(FieldType::DATE, 1), Literal(FieldType::DATE, -1)})); | |
| CheckLiteralFromRow(arrow::date32(), {0, 4, -5, NullType()}, FieldType::DATE, | |
| {Literal(FieldType::DATE, 0l), Literal(FieldType::DATE, 4l), | |
| Literal(FieldType::DATE, -5l), Literal(FieldType::DATE)}); | |
| } | |
| TEST_F(LiteralConverterTest, TestInvalidType) { | |
| auto field_array = | |
| arrow::ipc::internal::json::ArrayFromJSON(arrow::large_utf8(), R"(["apple", "苹果", null])") | |
| .ValueOrDie(); | |
| ASSERT_NOK_WITH_MSG( | |
| LiteralConverter::ConvertLiteralsFromArray(*field_array, /*own_data=*/false), | |
| "Not support literal on arrow large_string type"); | |
| } | |
| TEST_F(LiteralConverterTest, TestDictType) { | |
| auto dict = arrow::ipc::internal::json::ArrayFromJSON(arrow::utf8(), R"(["foo", "bar", "baz"])") | |
| .ValueOrDie(); | |
| auto dict_type = arrow::dictionary(arrow::int32(), arrow::utf8()); | |
| auto indices = | |
| arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), "[1, 2, 0, 2, 0, null]") | |
| .ValueOrDie(); | |
| std::shared_ptr<arrow::DictionaryArray> field_array = | |
| std::make_shared<arrow::DictionaryArray>(dict_type, indices, dict); | |
| CheckResult(field_array, | |
| std::vector<Literal>( | |
| {Literal(FieldType::STRING, "bar", 3), Literal(FieldType::STRING, "baz", 3), | |
| Literal(FieldType::STRING, "foo", 3), Literal(FieldType::STRING, "baz", 3), | |
| Literal(FieldType::STRING, "foo", 3), Literal(FieldType::STRING)})); | |
| } | |
| TEST_F(LiteralConverterTest, TestDictionaryValueAndIndexTypes) { | |
| const std::vector<std::pair<std::shared_ptr<arrow::DataType>, std::shared_ptr<arrow::DataType>>> | |
| types = {{arrow::utf8(), arrow::int32()}, | |
| {arrow::binary(), arrow::int32()}, | |
| {arrow::large_utf8(), arrow::int64()}, | |
| {arrow::large_binary(), arrow::int64()}}; | |
| for (const auto& [value_type, index_type] : types) { | |
| SCOPED_TRACE(value_type->ToString()); | |
| const FieldType literal_type = | |
| arrow::is_string(value_type->id()) ? FieldType::STRING : FieldType::BINARY; | |
| auto dictionary = arrow::ipc::internal::json::ArrayFromJSON( | |
| value_type, R"(["unused", "a\u0000b", "", "middle", "tail"])") | |
| .ValueOrDie() | |
| ->Slice(1); | |
| const std::vector<Literal> expected = { | |
| Literal(literal_type, "a\0b", 3), Literal(literal_type), | |
| Literal(literal_type, "", 0), Literal(literal_type, "middle", 6), | |
| Literal(literal_type, "tail", 4), Literal(literal_type, "a\0b", 3)}; | |
| SCOPED_TRACE(index_type->ToString()); | |
| auto indices = | |
| arrow::ipc::internal::json::ArrayFromJSON(index_type, "[3, 0, null, 1, 2, 3, 0]") | |
| .ValueOrDie(); | |
| auto array = arrow::DictionaryArray::FromArrays(indices, dictionary).ValueOrDie(); | |
| ASSERT_TRUE(array->ValidateFull().ok()); | |
| auto sliced = array->Slice(1); | |
| for (bool own_data : {false, true}) { | |
| SCOPED_TRACE(own_data); | |
| ASSERT_OK_AND_ASSIGN(std::vector<Literal> actual, | |
| LiteralConverter::ConvertLiteralsFromArray(*sliced, own_data)); | |
| ASSERT_EQ(expected, actual); | |
| } | |
| } | |
| } | |
| TEST_F(LiteralConverterTest, TestBinaryDictionaryPreservesBytes) { | |
| const std::string bytes("\x00\xff\x80", 3); | |
| arrow::BinaryBuilder builder; | |
| ASSERT_TRUE(builder.Append(bytes).ok()); | |
| std::shared_ptr<arrow::Array> dictionary; | |
| ASSERT_TRUE(builder.Finish(&dictionary).ok()); | |
| auto indices = | |
| arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), "[0, null, 0]").ValueOrDie(); | |
| auto array = arrow::DictionaryArray::FromArrays(indices, dictionary).ValueOrDie(); | |
| ASSERT_OK_AND_ASSIGN(std::vector<Literal> actual, | |
| LiteralConverter::ConvertLiteralsFromArray(*array, /*own_data=*/true)); | |
| array.reset(); | |
| dictionary.reset(); | |
| ASSERT_EQ(actual, | |
| std::vector<Literal>({Literal(FieldType::BINARY, bytes.data(), bytes.size()), | |
| Literal(FieldType::BINARY), | |
| Literal(FieldType::BINARY, bytes.data(), bytes.size())})); | |
| } | |
| TEST_F(LiteralConverterTest, TestLiteralsToArray) { | |
| // Every writable field type converts to an array and back, the null literal included, so each | |
| // case asserts `ConvertLiteralsToArray` and `ConvertLiteralsFromArray` agree on every type. | |
| CheckLiteralsToArray( | |
| FieldType::BOOLEAN, {Literal(true), Literal(FieldType::BOOLEAN), Literal(false)}, | |
| arrow::ipc::internal::json::ArrayFromJSON(arrow::boolean(), R"([true, null, false])") | |
| .ValueOrDie()); | |
| CheckLiteralsToArray( | |
| FieldType::TINYINT, | |
| {Literal(static_cast<int8_t>(4)), Literal(FieldType::TINYINT), | |
| Literal(static_cast<int8_t>(-5))}, | |
| arrow::ipc::internal::json::ArrayFromJSON(arrow::int8(), R"([4, null, -5])").ValueOrDie()); | |
| CheckLiteralsToArray( | |
| FieldType::SMALLINT, | |
| {Literal(static_cast<int16_t>(45)), Literal(FieldType::SMALLINT), | |
| Literal(static_cast<int16_t>(-55))}, | |
| arrow::ipc::internal::json::ArrayFromJSON(arrow::int16(), R"([45, null, -55])") | |
| .ValueOrDie()); | |
| CheckLiteralsToArray( | |
| FieldType::INT, {Literal(456), Literal(FieldType::INT), Literal(-567)}, | |
| arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), R"([456, null, -567])") | |
| .ValueOrDie()); | |
| CheckLiteralsToArray( | |
| FieldType::BIGINT, {Literal(4l), Literal(FieldType::BIGINT), Literal(-5l)}, | |
| arrow::ipc::internal::json::ArrayFromJSON(arrow::int64(), R"([4, null, -5])").ValueOrDie()); | |
| CheckLiteralsToArray( | |
| FieldType::FLOAT, {Literal(4.5f), Literal(FieldType::FLOAT), Literal(-5.25f)}, | |
| arrow::ipc::internal::json::ArrayFromJSON(arrow::float32(), R"([4.5, null, -5.25])") | |
| .ValueOrDie()); | |
| CheckLiteralsToArray( | |
| FieldType::DOUBLE, {Literal(4.5), Literal(FieldType::DOUBLE), Literal(-5.25)}, | |
| arrow::ipc::internal::json::ArrayFromJSON(arrow::float64(), R"([4.5, null, -5.25])") | |
| .ValueOrDie()); | |
| CheckLiteralsToArray( | |
| FieldType::DATE, | |
| {Literal(FieldType::DATE, 0), Literal(FieldType::DATE), Literal(FieldType::DATE, -5)}, | |
| arrow::ipc::internal::json::ArrayFromJSON(arrow::date32(), R"([0, null, -5])") | |
| .ValueOrDie()); | |
| std::string str = "苹果"; | |
| CheckLiteralsToArray( | |
| FieldType::STRING, | |
| {Literal(FieldType::STRING, "apple", 5), Literal(FieldType::STRING), | |
| Literal(FieldType::STRING, str.data(), str.size())}, | |
| arrow::ipc::internal::json::ArrayFromJSON(arrow::utf8(), R"(["apple", null, "苹果"])") | |
| .ValueOrDie()); | |
| CheckLiteralsToArray( | |
| FieldType::BINARY, | |
| {Literal(FieldType::BINARY, "apple", 5), Literal(FieldType::BINARY), | |
| Literal(FieldType::BINARY, str.data(), str.size())}, | |
| arrow::ipc::internal::json::ArrayFromJSON(arrow::binary(), R"(["apple", null, "苹果"])") | |
| .ValueOrDie()); | |
| // a decimal is written with the precision and the scale its literals carry | |
| CheckLiteralsToArray( | |
| FieldType::DECIMAL, | |
| {Literal(Decimal(21, 3, DecimalUtils::StrToInt128("-123456789987654321234").value())), | |
| Literal(FieldType::DECIMAL), Literal(Decimal(21, 3, 123456))}, | |
| arrow::ipc::internal::json::ArrayFromJSON(arrow::decimal128(21, 3), | |
| R"(["-123456789987654321.234", null, "123.456"])") | |
| .ValueOrDie()); | |
| // a timestamp is written with the finest time unit that keeps the values, one per case, and | |
| // the JSON reader takes a raw integer in the unit of the type it is given | |
| CheckLiteralsToArray( | |
| FieldType::TIMESTAMP, | |
| {Literal(Timestamp::FromEpochMillis(123000l)), Literal(FieldType::TIMESTAMP), | |
| Literal(Timestamp::FromEpochMillis(-456000l))}, | |
| arrow::ipc::internal::json::ArrayFromJSON(arrow::timestamp(arrow::TimeUnit::MILLI), | |
| R"([123000, null, -456000])") | |
| .ValueOrDie()); | |
| CheckLiteralsToArray( | |
| FieldType::TIMESTAMP, | |
| {Literal(Timestamp(123000l, 1000)), Literal(FieldType::TIMESTAMP), | |
| Literal(Timestamp(-456000l, 2000))}, | |
| arrow::ipc::internal::json::ArrayFromJSON(arrow::timestamp(arrow::TimeUnit::MICRO), | |
| R"([123000001, null, -455999998])") | |
| .ValueOrDie()); | |
| CheckLiteralsToArray( | |
| FieldType::TIMESTAMP, | |
| {Literal(Timestamp(123000l, 456789)), Literal(FieldType::TIMESTAMP), | |
| Literal(Timestamp(-456000l, 1))}, | |
| arrow::ipc::internal::json::ArrayFromJSON(arrow::timestamp(arrow::TimeUnit::NANO), | |
| R"([123000456789, null, -455999999999])") | |
| .ValueOrDie()); | |
| } | |
| TEST_F(LiteralConverterTest, TestLiteralsToArrayWithoutValue) { | |
| // no literal at all still gives an array typed after the field type | |
| CheckLiteralsToArray( | |
| FieldType::INT, {}, | |
| arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), R"([])").ValueOrDie()); | |
| // and a field type is all a null literal needs to be written | |
| CheckLiteralsToArray( | |
| FieldType::STRING, {Literal(FieldType::STRING), Literal(FieldType::STRING)}, | |
| arrow::ipc::internal::json::ArrayFromJSON(arrow::utf8(), R"([null, null])").ValueOrDie()); | |
| } | |
| TEST_F(LiteralConverterTest, TestLiteralsToArrayUnsupportedType) { | |
| // a timestamp takes its arrow type from a literal, so it needs one that is not null | |
| ASSERT_NOK_WITH_MSG(LiteralConverter::ConvertLiteralsToArray(FieldType::TIMESTAMP, {}, | |
| arrow::default_memory_pool()), | |
| "without a non null literal"); | |
| ASSERT_NOK_WITH_MSG( | |
| LiteralConverter::ConvertLiteralsToArray( | |
| FieldType::TIMESTAMP, {Literal(FieldType::TIMESTAMP)}, arrow::default_memory_pool()), | |
| "without a non null literal"); | |
| // a decimal takes its arrow type from a literal, so it needs one that is not null | |
| ASSERT_NOK_WITH_MSG(LiteralConverter::ConvertLiteralsToArray(FieldType::DECIMAL, {}, | |
| arrow::default_memory_pool()), | |
| "without a non null literal"); | |
| ASSERT_NOK_WITH_MSG( | |
| LiteralConverter::ConvertLiteralsToArray(FieldType::DECIMAL, {Literal(FieldType::DECIMAL)}, | |
| arrow::default_memory_pool()), | |
| "without a non null literal"); | |
| // and every non null literal has to carry the same precision and scale | |
| ASSERT_NOK_WITH_MSG( | |
| LiteralConverter::ConvertLiteralsToArray( | |
| FieldType::DECIMAL, {Literal(Decimal(21, 3, 123456)), Literal(Decimal(21, 5, 123456))}, | |
| arrow::default_memory_pool()), | |
| "do not share one precision and scale"); | |
| ASSERT_NOK_WITH_MSG( | |
| LiteralConverter::ConvertLiteralsToArray( | |
| FieldType::DECIMAL, {Literal(Decimal(21, 3, 123456)), Literal(Decimal(20, 3, 123456))}, | |
| arrow::default_memory_pool()), | |
| "do not share one precision and scale"); | |
| // a precision arrow rejects is reported instead of checked fatally | |
| ASSERT_NOK_WITH_MSG( | |
| LiteralConverter::ConvertLiteralsToArray(FieldType::DECIMAL, {Literal(Decimal(0, 0, 1))}, | |
| arrow::default_memory_pool()), | |
| "Decimal precision out of range"); | |
| // the field types no caller asks for yet | |
| ASSERT_NOK_WITH_MSG( | |
| LiteralConverter::ConvertLiteralsToArray(FieldType::BLOB, {}, arrow::default_memory_pool()), | |
| "Not support converting literals of BLOB type to an arrow array"); | |
| ASSERT_NOK_WITH_MSG(LiteralConverter::ConvertLiteralsToArray(FieldType::ARRAY, {}, | |
| arrow::default_memory_pool()), | |
| "Not support converting literals of ARRAY type to an arrow array"); | |
| ASSERT_NOK_WITH_MSG( | |
| LiteralConverter::ConvertLiteralsToArray(FieldType::MAP, {}, arrow::default_memory_pool()), | |
| "Not support converting literals of MAP type to an arrow array"); | |
| ASSERT_NOK_WITH_MSG(LiteralConverter::ConvertLiteralsToArray(FieldType::STRUCT, {}, | |
| arrow::default_memory_pool()), | |
| "Not support converting literals of STRUCT type to an arrow array"); | |
| ASSERT_NOK_WITH_MSG(LiteralConverter::ConvertLiteralsToArray(FieldType::UNKNOWN, {}, | |
| arrow::default_memory_pool()), | |
| "Not support converting literals of UNKNOWN"); | |
| } | |
| } // namespace paimon::test |