| /* |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, |
| * software distributed under the License is distributed on an |
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| * KIND, either express or implied. See the License for the |
| * specific language governing permissions and limitations |
| * under the License. |
| */ |
| |
| #include "paimon/common/data/variant/generic_variant.h" |
| |
| #include <functional> |
| #include <limits> |
| #include <string> |
| #include <vector> |
| |
| #include "gtest/gtest.h" |
| #include "paimon/common/data/variant/variant_builder.h" |
| #include "paimon/common/data/variant/variant_defs.h" |
| #include "paimon/memory/memory_pool.h" |
| #include "paimon/testing/utils/testharness.h" |
| |
| namespace paimon::test { |
| |
| class GenericVariantTest : public ::testing::Test { |
| public: |
| static std::string ToHex(std::string_view data) { |
| constexpr char kHexDigits[] = "0123456789abcdef"; |
| std::string result; |
| result.reserve(data.size() * 2); |
| for (char c : data) { |
| auto byte = static_cast<uint8_t>(c); |
| result.push_back(kHexDigits[byte >> 4]); |
| result.push_back(kHexDigits[byte & 0xF]); |
| } |
| return result; |
| } |
| |
| std::shared_ptr<GenericVariant> FromJson(const std::string& json) { |
| auto result = GenericVariant::FromJson(json, pool_); |
| EXPECT_TRUE(result.ok()) << result.status().ToString(); |
| return result.value(); |
| } |
| |
| // Asserts that parsing `json` produces exactly the value/metadata binaries produced by the |
| // Java implementation (`GenericVariantBuilder`), and that rendering back to JSON matches the |
| // Java `toJson` output. |
| void CheckGolden(const std::string& json, const std::string& expected_value_hex, |
| const std::string& expected_metadata_hex, |
| const std::string& expected_to_json) { |
| auto variant = FromJson(json); |
| ASSERT_OK_AND_ASSIGN(std::string_view value, variant->Value()); |
| ASSERT_EQ(ToHex(value), expected_value_hex) << "value bytes mismatch for: " << json; |
| ASSERT_EQ(ToHex(variant->Metadata()), expected_metadata_hex) |
| << "metadata bytes mismatch for: " << json; |
| ASSERT_OK_AND_ASSIGN(std::string to_json, variant->ToJson()); |
| ASSERT_EQ(to_json, expected_to_json); |
| } |
| |
| protected: |
| std::shared_ptr<MemoryPool> pool_ = GetDefaultPool(); |
| }; |
| |
| // Golden binaries generated by the Java implementation (org.apache.paimon.data.variant); these |
| // pin the cross-implementation byte compatibility of the variant encoding. |
| TEST_F(GenericVariantTest, GoldenPrimitives) { |
| CheckGolden("null", "00", "010000", "null"); |
| CheckGolden("true", "04", "010000", "true"); |
| CheckGolden("false", "08", "010000", "false"); |
| CheckGolden("1", "0c01", "010000", "1"); |
| CheckGolden("-1", "0cff", "010000", "-1"); |
| CheckGolden("300", "102c01", "010000", "300"); |
| CheckGolden("100000", "14a0860100", "010000", "100000"); |
| CheckGolden("12345678901234", "18f22fce733a0b0000", "010000", "12345678901234"); |
| CheckGolden("1e40", "1ca55cc3f129633d48", "010000", "1.0E40"); |
| CheckGolden("1.0123456789012345678901234567890123456789", "1c240bf2619132f03f", "010000", |
| "1.0123456789012346"); |
| CheckGolden("2.5e-3", "1c7b14ae47e17a643f", "010000", "0.0025"); |
| CheckGolden("100.99", "200273270000", "010000", "100.99"); |
| CheckGolden("-0.5", "2001fbffffff", "010000", "-0.5"); |
| CheckGolden("0.0", "200100000000", "010000", "0"); |
| CheckGolden("12345678.90123", "2405cb04fb711f010000", "010000", "12345678.90123"); |
| CheckGolden("1234567890123456789.0123456789", "280a1581396eb1c9be46321be42700000000", "010000", |
| "1234567890123456789.0123456789"); |
| // An integer that overflows int64 is parsed as an exact decimal. |
| CheckGolden("123456789012345678901234567890", "2800d20a3f4eeee073c3f60fe98e01000000", "010000", |
| "123456789012345678901234567890"); |
| } |
| |
| TEST_F(GenericVariantTest, GoldenStrings) { |
| CheckGolden("\"Hello, World!\"", "3548656c6c6f2c20576f726c6421", "010000", "\"Hello, World!\""); |
| CheckGolden("\"\"", "01", "010000", "\"\""); |
| CheckGolden( |
| "\"This is a long string that definitely exceeds the sixty-three byte short string limit " |
| "...!\"", |
| "405a000000546869732069732061206c6f6e6720737472696e67207468617420646566696e6974656c792065" |
| "786365656473207468652073697874792d746872656520627974652073686f727420737472696e67206c696d" |
| "6974202e2e2e21", |
| "010000", |
| "\"This is a long string that definitely exceeds the sixty-three byte short string limit " |
| "...!\""); |
| } |
| |
| TEST_F(GenericVariantTest, GoldenContainers) { |
| CheckGolden("{}", "020000", "010000", "{}"); |
| CheckGolden("[]", "030000", "010000", "[]"); |
| CheckGolden(R"({"a": 1, "b": "hello"})", "020200010002080c011568656c6c6f", "01020001026162", |
| R"({"a":1,"b":"hello"})"); |
| CheckGolden(R"([1, "two", 3.5, null, true, {"k":[]}])", |
| "03060002060c0d0e160c010d74776f20012300000000040201000003030000", "010100016b", |
| R"([1,"two",3.5,null,true,{"k":[]}])"); |
| } |
| |
| TEST_F(GenericVariantTest, GoldenNested) { |
| CheckGolden( |
| "{\"object\":{\"name\":\"Apache Paimon\",\"age\":2,\"address\":{\"street\":\"Main " |
| "St\",\"city\":\"Hangzhou\"}},\"array\":[1,2,3,4,5],\"string\":\"Hello, " |
| "World!\",\"long\":12345678901234,\"double\":1." |
| "0123456789012345678901234567890123456789,\"decimal\":100.99,\"boolean1\":true," |
| "\"boolean2\":false,\"nullField\":null}", |
| "0209060b0c0a09080d000731696a635a516b00436c0203030201100e0028354170616368652050" |
| "61696d6f6e0c02020205040800111d4d61696e2053742148616e677a686f75030500020406080a" |
| "0c010c020c030c040c053548656c6c6f2c20576f726c642118f22fce733a0b00001c240bf26191" |
| "32f03f200273270000040800", |
| "010e00060a0d141a1e23292d333a424a536f626a6563746e616d6561676561646472657373737472656574" |
| "636974796172726179737472696e676c6f6e67646f75626c65646563696d616c626f6f6c65616e31626f6f" |
| "6c65616e326e756c6c4669656c64", |
| "{\"array\":[1,2,3,4,5],\"boolean1\":true,\"boolean2\":false,\"decimal\":100.99," |
| "\"double\":1.0123456789012346,\"long\":12345678901234,\"nullField\":null,\"object\":{" |
| "\"address\":{\"city\":\"Hangzhou\",\"street\":\"Main St\"},\"age\":2,\"name\":\"Apache " |
| "Paimon\"},\"string\":\"Hello, World!\"}"); |
| } |
| |
| TEST_F(GenericVariantTest, GoldenUnicodeEscape) { |
| CheckGolden(R"({"\u4e2d\u6587": "\u4f60\u597d\n\t\"quoted\""})", |
| "020100001141e4bda0e5a5bd0a092271756f74656422", "01010006e4b8ade69687", |
| "{\"中文\":\"你好\\n\\t\\\"quoted\\\"\"}"); |
| } |
| |
| TEST_F(GenericVariantTest, TypedAccessors) { |
| auto variant = FromJson(R"({"a": 1, "b": "hello"})"); |
| ASSERT_OK_AND_ASSIGN(VariantValueType type, variant->GetType()); |
| ASSERT_EQ(type, VariantValueType::kObject); |
| ASSERT_OK_AND_ASSIGN(int32_t object_size, variant->ObjectSize()); |
| ASSERT_EQ(object_size, 2); |
| |
| ASSERT_OK_AND_ASSIGN(std::shared_ptr<GenericVariant> a, variant->GetFieldByKey("a")); |
| ASSERT_NE(a, nullptr); |
| ASSERT_OK_AND_ASSIGN(int64_t a_value, a->GetLong()); |
| ASSERT_EQ(a_value, 1); |
| |
| ASSERT_OK_AND_ASSIGN(std::shared_ptr<GenericVariant> b, variant->GetFieldByKey("b")); |
| ASSERT_NE(b, nullptr); |
| ASSERT_OK_AND_ASSIGN(std::string_view b_value, b->GetString()); |
| ASSERT_EQ(b_value, "hello"); |
| |
| ASSERT_OK_AND_ASSIGN(std::shared_ptr<GenericVariant> missing, variant->GetFieldByKey("c")); |
| ASSERT_EQ(missing, nullptr); |
| |
| ASSERT_OK_AND_ASSIGN(auto field0, variant->GetFieldAtIndex(0)); |
| ASSERT_TRUE(field0.has_value()); |
| ASSERT_EQ(field0->key, "a"); |
| ASSERT_OK_AND_ASSIGN(auto field_oob, variant->GetFieldAtIndex(2)); |
| ASSERT_FALSE(field_oob.has_value()); |
| |
| auto array = FromJson("[10, 20, 30]"); |
| ASSERT_OK_AND_ASSIGN(int32_t array_size, array->ArraySize()); |
| ASSERT_EQ(array_size, 3); |
| ASSERT_OK_AND_ASSIGN(std::shared_ptr<GenericVariant> elem, array->GetElementAtIndex(1)); |
| ASSERT_NE(elem, nullptr); |
| ASSERT_OK_AND_ASSIGN(int64_t elem_value, elem->GetLong()); |
| ASSERT_EQ(elem_value, 20); |
| ASSERT_OK_AND_ASSIGN(std::shared_ptr<GenericVariant> elem_oob, array->GetElementAtIndex(3)); |
| ASSERT_EQ(elem_oob, nullptr); |
| } |
| |
| TEST_F(GenericVariantTest, ObjectBinarySearch) { |
| // More fields than kBinarySearchThreshold exercises the binary-search lookup. |
| std::string json = "{"; |
| for (int32_t i = 0; i < 40; ++i) { |
| if (i != 0) { |
| json += ","; |
| } |
| json += "\"key" + std::to_string(i) + "\":" + std::to_string(i); |
| } |
| json += "}"; |
| auto variant = FromJson(json); |
| for (int32_t i = 0; i < 40; ++i) { |
| ASSERT_OK_AND_ASSIGN(std::shared_ptr<GenericVariant> field, |
| variant->GetFieldByKey("key" + std::to_string(i))); |
| ASSERT_NE(field, nullptr); |
| ASSERT_OK_AND_ASSIGN(int64_t value, field->GetLong()); |
| ASSERT_EQ(value, i); |
| } |
| ASSERT_OK_AND_ASSIGN(std::shared_ptr<GenericVariant> missing, variant->GetFieldByKey("key40")); |
| ASSERT_EQ(missing, nullptr); |
| } |
| |
| TEST_F(GenericVariantTest, DuplicateKeys) { |
| ASSERT_NOK(GenericVariant::FromJson("{\"a\": 1, \"a\": 2}", pool_)); |
| ASSERT_OK_AND_ASSIGN( |
| std::shared_ptr<GenericVariant> variant, |
| VariantBuilder::ParseJson("{\"a\": 1, \"a\": 2}", /*allow_duplicate_keys=*/true, pool_)); |
| ASSERT_OK_AND_ASSIGN(std::string to_json, variant->ToJson()); |
| ASSERT_EQ(to_json, "{\"a\":2}"); |
| } |
| |
| TEST_F(GenericVariantTest, CorruptedLayoutRejected) { |
| // Headers claiming a near-INT32_MAX element count must be rejected by the 64-bit layout |
| // bound instead of overflowing the 32-bit offset arithmetic. |
| std::string metadata; |
| metadata.push_back(static_cast<char>(0x01)); |
| metadata.push_back(static_cast<char>(0x00)); |
| { |
| // Object, large size, 4-byte ids and offsets, num_elements = INT32_MAX. |
| std::string value; |
| value.push_back(static_cast<char>((0x1F << 2) | 0x02)); |
| value.append({static_cast<char>(0xFF), static_cast<char>(0xFF), static_cast<char>(0xFF), |
| static_cast<char>(0x7F)}); |
| ASSERT_OK_AND_ASSIGN(std::shared_ptr<GenericVariant> variant, |
| GenericVariant::Create(value, metadata, pool_)); |
| ASSERT_NOK(variant->ToJson()); |
| } |
| { |
| // Array, large size, 4-byte offsets, num_elements = INT32_MAX. |
| std::string value; |
| value.push_back(static_cast<char>((0x07 << 2) | 0x03)); |
| value.append({static_cast<char>(0xFF), static_cast<char>(0xFF), static_cast<char>(0xFF), |
| static_cast<char>(0x7F)}); |
| ASSERT_OK_AND_ASSIGN(std::shared_ptr<GenericVariant> variant, |
| GenericVariant::Create(value, metadata, pool_)); |
| ASSERT_NOK(variant->ToJson()); |
| } |
| } |
| |
| TEST_F(GenericVariantTest, OverLimitInputRejectedBeforeAllocation) { |
| // The size check must run before the buffers are allocated/copied; the fake-length view is |
| // never dereferenced. |
| char byte = 0; |
| std::string_view huge(&byte, static_cast<size_t>(VariantDefs::kSizeLimit) + 1); |
| ASSERT_NOK(GenericVariant::Create(huge, std::string_view(&byte, 1), pool_)); |
| ASSERT_NOK(GenericVariant::Create(std::string_view(&byte, 1), huge, pool_)); |
| } |
| |
| TEST_F(GenericVariantTest, MalformedInput) { |
| ASSERT_NOK(GenericVariant::FromJson("", pool_)); |
| ASSERT_NOK(GenericVariant::FromJson("{", pool_)); |
| ASSERT_NOK(GenericVariant::FromJson("{\"a\":}", pool_)); |
| ASSERT_NOK(GenericVariant::FromJson("NaN", pool_)); |
| |
| // Unsupported metadata version. |
| std::string bad_metadata = std::string("\x02\x00\x00", 3); |
| std::string value = std::string("\x00", 1); |
| ASSERT_NOK(GenericVariant::Create(value, bad_metadata, pool_)); |
| // Empty metadata. |
| ASSERT_NOK(GenericVariant::Create(value, std::string(), pool_)); |
| } |
| |
| TEST_F(GenericVariantTest, SizeInBytesAndViews) { |
| auto variant = FromJson(R"({"a": 1, "b": "hello"})"); |
| ASSERT_EQ(variant->SizeInBytes(), |
| static_cast<int64_t>(variant->RawValue().size() + variant->Metadata().size())); |
| // A sub-variant shares buffers with its parent and reports a positive position. |
| ASSERT_OK_AND_ASSIGN(std::shared_ptr<GenericVariant> b, variant->GetFieldByKey("b")); |
| ASSERT_GT(b->Pos(), 0); |
| ASSERT_OK_AND_ASSIGN(std::string_view b_slice, b->Value()); |
| ASSERT_EQ(ToHex(b_slice), "1568656c6c6f"); |
| } |
| |
| TEST_F(GenericVariantTest, AppendVariantRebuild) { |
| // Rebuilding a sub-variant through a fresh builder produces a self-contained variant. |
| auto variant = FromJson(R"({"outer": {"x": [1, 2], "y": "z"}})"); |
| ASSERT_OK_AND_ASSIGN(std::shared_ptr<GenericVariant> outer, variant->GetFieldByKey("outer")); |
| VariantBuilder builder(/*allow_duplicate_keys=*/false); |
| ASSERT_OK(builder.AppendVariant(*outer)); |
| ASSERT_OK_AND_ASSIGN(std::shared_ptr<GenericVariant> rebuilt, builder.Build(pool_)); |
| ASSERT_EQ(rebuilt->Pos(), 0); |
| ASSERT_OK_AND_ASSIGN(std::string to_json, rebuilt->ToJson()); |
| ASSERT_EQ(to_json, "{\"x\":[1,2],\"y\":\"z\"}"); |
| } |
| |
| TEST_F(GenericVariantTest, TimestampAndSpecialTypesToJson) { |
| // JSON can't produce date/timestamp/binary/uuid variants; build them directly. |
| { |
| VariantBuilder builder(false); |
| ASSERT_OK(builder.AppendDate(19737)); // 2024-01-15 |
| ASSERT_OK_AND_ASSIGN(auto v, builder.Build(pool_)); |
| ASSERT_OK_AND_ASSIGN(std::string json, v->ToJson()); |
| ASSERT_EQ(json, "\"2024-01-15\""); |
| } |
| { |
| VariantBuilder builder(false); |
| ASSERT_OK(builder.AppendTimestamp(1705312496123456LL)); |
| ASSERT_OK_AND_ASSIGN(auto v, builder.Build(pool_)); |
| ASSERT_OK_AND_ASSIGN(std::string json, v->ToJson()); |
| ASSERT_EQ(json, "\"2024-01-15 09:54:56.123456+00:00\""); |
| ASSERT_OK_AND_ASSIGN(std::string json_shanghai, v->ToJson("Asia/Shanghai")); |
| ASSERT_EQ(json_shanghai, "\"2024-01-15 17:54:56.123456+08:00\""); |
| ASSERT_OK_AND_ASSIGN(std::string json_offset, v->ToJson("+08:00")); |
| ASSERT_EQ(json_offset, "\"2024-01-15 17:54:56.123456+08:00\""); |
| ASSERT_NOK(v->ToJson("Not/AZone")); |
| } |
| { |
| VariantBuilder builder(false); |
| ASSERT_OK(builder.AppendTimestampNtz(1705312496000000LL)); |
| ASSERT_OK_AND_ASSIGN(auto v, builder.Build(pool_)); |
| ASSERT_OK_AND_ASSIGN(std::string json, v->ToJson()); |
| ASSERT_EQ(json, "\"2024-01-15 09:54:56\""); |
| } |
| { |
| VariantBuilder builder(false); |
| ASSERT_OK(builder.AppendBinary(std::string_view("\x01\x02\x03", 3))); |
| ASSERT_OK_AND_ASSIGN(auto v, builder.Build(pool_)); |
| ASSERT_OK_AND_ASSIGN(std::string json, v->ToJson()); |
| ASSERT_EQ(json, "\"AQID\""); |
| } |
| { |
| VariantBuilder builder(false); |
| std::string uuid_bytes = |
| std::string("\x12\x3e\x45\x67\xe8\x9b\x12\xd3\xa4\x56\x42\x66\x14\x17\x40\x00", 16); |
| ASSERT_OK(builder.AppendUuid(uuid_bytes)); |
| ASSERT_OK_AND_ASSIGN(auto v, builder.Build(pool_)); |
| ASSERT_OK_AND_ASSIGN(std::string json, v->ToJson()); |
| ASSERT_EQ(json, "\"123e4567-e89b-12d3-a456-426614174000\""); |
| } |
| { |
| VariantBuilder builder(false); |
| ASSERT_OK(builder.AppendFloat(1.5f)); |
| ASSERT_OK_AND_ASSIGN(auto v, builder.Build(pool_)); |
| ASSERT_OK_AND_ASSIGN(std::string json, v->ToJson()); |
| ASSERT_EQ(json, "1.5"); |
| } |
| } |
| |
| TEST_F(GenericVariantTest, NonFiniteDoubleToJson) { |
| VariantBuilder builder(false); |
| ASSERT_OK(builder.AppendDouble(std::numeric_limits<double>::infinity())); |
| ASSERT_OK_AND_ASSIGN(auto v, builder.Build(pool_)); |
| ASSERT_OK_AND_ASSIGN(std::string json, v->ToJson()); |
| ASSERT_EQ(json, "\"Infinity\""); |
| } |
| |
| TEST_F(GenericVariantTest, GetTypeInfoReturnsHeaderBits) { |
| // GetTypeInfo exposes the primitive header's type-info bits; 42 is encoded as an int1. |
| auto v = FromJson("42"); |
| ASSERT_OK_AND_ASSIGN(int32_t type_info, v->GetTypeInfo()); |
| EXPECT_EQ(type_info, VariantDefs::kInt1); |
| } |
| |
| TEST_F(GenericVariantTest, TypedGettersRejectMismatchedTypes) { |
| // Each accessor validates the value header and fails when the stored type differs. |
| auto number = FromJson("42"); // primitive int1 |
| auto text = FromJson("\"hi\""); // short string |
| auto real = FromJson("1.5e0"); // double |
| |
| // A primitive long is neither boolean/double/decimal/float/binary/string/uuid, nor a |
| // container. |
| ASSERT_NOK(number->GetBoolean()); |
| ASSERT_NOK(number->GetDouble()); |
| ASSERT_NOK(number->GetDecimal()); |
| ASSERT_NOK(number->GetFloat()); |
| ASSERT_NOK(number->GetBinary()); |
| ASSERT_NOK(number->GetString()); |
| ASSERT_NOK(number->GetUuid()); |
| ASSERT_NOK(number->ObjectSize()); |
| ASSERT_NOK(number->ArraySize()); |
| // A short string is not a primitive, so long/decimal decoding rejects it early. |
| ASSERT_NOK(text->GetLong()); |
| ASSERT_NOK(text->GetDecimal()); |
| // A double is a primitive but not an integer-like type. |
| ASSERT_NOK(real->GetLong()); |
| } |
| |
| TEST_F(GenericVariantTest, ValueSizeCoversAllPrimitiveWidths) { |
| // Builds an array whose elements span the primitive width branches of `ValueSize` (int4, |
| // decimal8, binary, uuid). Copying elements into the array and re-reading each element's |
| // value both exercise `ValueSize`. |
| auto build = [this](const std::function<Status(VariantBuilder&)>& append) { |
| VariantBuilder builder(/*allow_duplicate_keys=*/false); |
| EXPECT_OK(append(builder)); |
| auto result = builder.Build(pool_); |
| EXPECT_TRUE(result.ok()) << result.status().ToString(); |
| return result.value(); |
| }; |
| std::shared_ptr<GenericVariant> int4 = |
| build([](VariantBuilder& b) { return b.AppendLong(100000); }); // needs 4 bytes |
| std::shared_ptr<GenericVariant> decimal8 = |
| build([](VariantBuilder& b) { return b.AppendDecimal(VariantDecimal{1234567890, 2}); }); |
| std::shared_ptr<GenericVariant> binary = |
| build([](VariantBuilder& b) { return b.AppendBinary(std::string_view("abc", 3)); }); |
| std::string uuid_bytes(16, '\x07'); |
| std::shared_ptr<GenericVariant> uuid = |
| build([&](VariantBuilder& b) { return b.AppendUuid(uuid_bytes); }); |
| |
| VariantBuilder array_builder(/*allow_duplicate_keys=*/false); |
| int32_t start = array_builder.GetWritePos(); |
| std::vector<int32_t> offsets; |
| for (const std::shared_ptr<GenericVariant>* element : {&int4, &decimal8, &binary, &uuid}) { |
| offsets.push_back(array_builder.GetWritePos() - start); |
| ASSERT_OK(array_builder.AppendVariant(**element)); |
| } |
| ASSERT_OK(array_builder.FinishWritingArray(start, offsets)); |
| ASSERT_OK_AND_ASSIGN(std::shared_ptr<GenericVariant> array_variant, array_builder.Build(pool_)); |
| |
| ASSERT_OK_AND_ASSIGN(int32_t size, array_variant->ArraySize()); |
| ASSERT_EQ(size, 4); |
| for (int32_t i = 0; i < size; ++i) { |
| ASSERT_OK_AND_ASSIGN(std::shared_ptr<GenericVariant> element, |
| array_variant->GetElementAtIndex(i)); |
| ASSERT_OK(element->Value()); |
| } |
| } |
| |
| } // namespace paimon::test |
