| /* | |
| * Licensed to the Apache Software Foundation (ASF) under one | |
| * or more contributor license agreements. See the NOTICE file | |
| * distributed with this work for additional information | |
| * regarding copyright ownership. The ASF licenses this file | |
| * to you under the Apache License, Version 2.0 (the | |
| * "License"); you may not use this file except in compliance | |
| * with the License. You may obtain a copy of the License at | |
| * | |
| * http://www.apache.org/licenses/LICENSE-2.0 | |
| * | |
| * Unless required by applicable law or agreed to in writing, | |
| * software distributed under the License is distributed on an | |
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| * KIND, either express or implied. See the License for the | |
| * specific language governing permissions and limitations | |
| * under the License. | |
| */ | |
| /* This file is based on source code from the Spark Project (http://spark.apache.org/), licensed | |
| * by the Apache Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE | |
| * file distributed with this work for additional information regarding copyright ownership. */ | |
| #include "paimon/common/data/variant/variant_binary_util.h" | |
| #include <algorithm> | |
| #include <cstring> | |
| #include <limits> | |
| #include "fmt/format.h" | |
| #include "paimon/common/data/variant/variant_defs.h" | |
| namespace paimon { | |
| int32_t VariantDecimal::Precision() const { | |
| __uint128_t abs_value = | |
| unscaled < 0 ? -static_cast<__uint128_t>(unscaled) : static_cast<__uint128_t>(unscaled); | |
| int32_t digits = 1; | |
| while (abs_value >= 10) { | |
| abs_value /= 10; | |
| ++digits; | |
| } | |
| return digits; | |
| } | |
| VariantDecimal VariantDecimal::StripTrailingZeros() const { | |
| VariantDecimal result = *this; | |
| if (result.unscaled == 0) { | |
| result.scale = 0; | |
| return result; | |
| } | |
| while (result.unscaled % 10 == 0) { | |
| result.unscaled /= 10; | |
| --result.scale; | |
| } | |
| return result; | |
| } | |
| std::string VariantDecimal::ToPlainString() const { | |
| __uint128_t abs_value = | |
| unscaled < 0 ? -static_cast<__uint128_t>(unscaled) : static_cast<__uint128_t>(unscaled); | |
| std::string digits; | |
| if (abs_value == 0) { | |
| digits = "0"; | |
| } else { | |
| while (abs_value > 0) { | |
| digits.push_back(static_cast<char>('0' + static_cast<int32_t>(abs_value % 10))); | |
| abs_value /= 10; | |
| } | |
| std::reverse(digits.begin(), digits.end()); | |
| } | |
| std::string result; | |
| if (unscaled < 0) { | |
| result.push_back('-'); | |
| } | |
| if (scale <= 0) { | |
| result.append(digits); | |
| result.append(static_cast<size_t>(-scale), '0'); | |
| } else if (static_cast<size_t>(scale) < digits.size()) { | |
| result.append(digits, 0, digits.size() - scale); | |
| result.push_back('.'); | |
| result.append(digits, digits.size() - scale, scale); | |
| } else { | |
| result.append("0."); | |
| result.append(static_cast<size_t>(scale) - digits.size(), '0'); | |
| result.append(digits); | |
| } | |
| return result; | |
| } | |
| Status VariantBinaryUtil::MalformedVariant(const std::string& message) { | |
| if (message.empty()) { | |
| return Status::Invalid("MALFORMED_VARIANT"); | |
| } | |
| return Status::Invalid(fmt::format("MALFORMED_VARIANT: {}", message)); | |
| } | |
| Status VariantBinaryUtil::UnknownPrimitiveTypeInVariant(int32_t id) { | |
| return Status::Invalid(fmt::format("UNKNOWN_PRIMITIVE_TYPE_IN_VARIANT, id: {}", id)); | |
| } | |
| Status VariantBinaryUtil::VariantConstructorSizeLimit() { | |
| return Status::Invalid("VARIANT_CONSTRUCTOR_SIZE_LIMIT"); | |
| } | |
| Status VariantBinaryUtil::UnexpectedType(VariantValueType type) { | |
| static constexpr const char* kTypeNames[] = { | |
| "OBJECT", "ARRAY", "NULL", "BOOLEAN", "LONG", "STRING", "DOUBLE", | |
| "DECIMAL", "DATE", "TIMESTAMP", "TIMESTAMP_NTZ", "FLOAT", "BINARY", "UUID"}; | |
| return Status::Invalid( | |
| fmt::format("Expect type to be {}", kTypeNames[static_cast<int32_t>(type)])); | |
| } | |
| Status VariantBinaryUtil::CheckIndex(int32_t pos, int32_t length) { | |
| if (pos < 0 || pos >= length) { | |
| return MalformedVariant( | |
| fmt::format("index {} is out of bounds for a buffer of {} bytes", pos, length)); | |
| } | |
| return Status::OK(); | |
| } | |
| void VariantBinaryUtil::WriteLong(int64_t value, int32_t num_bytes, uint8_t* bytes, int32_t pos) { | |
| for (int32_t i = 0; i < num_bytes; ++i) { | |
| bytes[pos + i] = static_cast<uint8_t>((static_cast<uint64_t>(value) >> (8 * i)) & 0xFF); | |
| } | |
| } | |
| Result<int64_t> VariantBinaryUtil::ReadLong(std::string_view bytes, int32_t pos, | |
| int32_t num_bytes) { | |
| auto length = static_cast<int32_t>(bytes.size()); | |
| PAIMON_RETURN_NOT_OK(CheckIndex(pos, length)); | |
| PAIMON_RETURN_NOT_OK(CheckIndex(pos + num_bytes - 1, length)); | |
| uint64_t result = 0; | |
| // All bytes except the most significant byte should be unsign-extended and shifted. The most | |
| // significant byte should be sign-extended. | |
| for (int32_t i = 0; i < num_bytes - 1; ++i) { | |
| uint64_t unsigned_byte_value = static_cast<uint8_t>(bytes[pos + i]); | |
| result |= unsigned_byte_value << (8 * i); | |
| } | |
| int64_t signed_byte_value = static_cast<int8_t>(bytes[pos + num_bytes - 1]); | |
| result |= static_cast<uint64_t>(signed_byte_value) << (8 * (num_bytes - 1)); | |
| return static_cast<int64_t>(result); | |
| } | |
| Result<int32_t> VariantBinaryUtil::ReadUnsigned(std::string_view bytes, int32_t pos, | |
| int32_t num_bytes) { | |
| auto length = static_cast<int32_t>(bytes.size()); | |
| PAIMON_RETURN_NOT_OK(CheckIndex(pos, length)); | |
| PAIMON_RETURN_NOT_OK(CheckIndex(pos + num_bytes - 1, length)); | |
| int64_t result = 0; | |
| // Similar to the `ReadLong` loop, but all bytes should be unsign-extended. | |
| for (int32_t i = 0; i < num_bytes; ++i) { | |
| int64_t unsigned_byte_value = static_cast<uint8_t>(bytes[pos + i]); | |
| result |= unsigned_byte_value << (8 * i); | |
| } | |
| if (result < 0 || result > std::numeric_limits<int32_t>::max()) { | |
| return MalformedVariant(fmt::format("unsigned value {} does not fit into int32", result)); | |
| } | |
| return static_cast<int32_t>(result); | |
| } | |
| uint8_t VariantBinaryUtil::PrimitiveHeader(int32_t type) { | |
| return static_cast<uint8_t>(type << 2 | VariantDefs::kPrimitive); | |
| } | |
| uint8_t VariantBinaryUtil::ShortStrHeader(int32_t size) { | |
| return static_cast<uint8_t>(size << 2 | VariantDefs::kShortStr); | |
| } | |
| uint8_t VariantBinaryUtil::ObjectHeader(bool large_size, int32_t id_size, int32_t offset_size) { | |
| return static_cast<uint8_t>(((large_size ? 1 : 0) << (VariantDefs::kBasicTypeBits + 4)) | | |
| ((id_size - 1) << (VariantDefs::kBasicTypeBits + 2)) | | |
| ((offset_size - 1) << VariantDefs::kBasicTypeBits) | | |
| VariantDefs::kObject); | |
| } | |
| uint8_t VariantBinaryUtil::ArrayHeader(bool large_size, int32_t offset_size) { | |
| return static_cast<uint8_t>(((large_size ? 1 : 0) << (VariantDefs::kBasicTypeBits + 2)) | | |
| ((offset_size - 1) << VariantDefs::kBasicTypeBits) | | |
| VariantDefs::kArray); | |
| } | |
| Result<int32_t> VariantBinaryUtil::GetTypeInfo(std::string_view value, int32_t pos) { | |
| PAIMON_RETURN_NOT_OK(CheckIndex(pos, static_cast<int32_t>(value.size()))); | |
| return (static_cast<uint8_t>(value[pos]) >> VariantDefs::kBasicTypeBits) & | |
| VariantDefs::kTypeInfoMask; | |
| } | |
| Result<VariantValueType> VariantBinaryUtil::GetType(std::string_view value, int32_t pos) { | |
| PAIMON_RETURN_NOT_OK(CheckIndex(pos, static_cast<int32_t>(value.size()))); | |
| auto header = static_cast<uint8_t>(value[pos]); | |
| int32_t basic_type = header & VariantDefs::kBasicTypeMask; | |
| int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; | |
| switch (basic_type) { | |
| case VariantDefs::kShortStr: | |
| return VariantValueType::kString; | |
| case VariantDefs::kObject: | |
| return VariantValueType::kObject; | |
| case VariantDefs::kArray: | |
| return VariantValueType::kArray; | |
| default: | |
| switch (type_info) { | |
| case VariantDefs::kNull: | |
| return VariantValueType::kNull; | |
| case VariantDefs::kTrue: | |
| case VariantDefs::kFalse: | |
| return VariantValueType::kBoolean; | |
| case VariantDefs::kInt1: | |
| case VariantDefs::kInt2: | |
| case VariantDefs::kInt4: | |
| case VariantDefs::kInt8: | |
| return VariantValueType::kLong; | |
| case VariantDefs::kDouble: | |
| return VariantValueType::kDouble; | |
| case VariantDefs::kDecimal4: | |
| case VariantDefs::kDecimal8: | |
| case VariantDefs::kDecimal16: | |
| return VariantValueType::kDecimal; | |
| case VariantDefs::kDate: | |
| return VariantValueType::kDate; | |
| case VariantDefs::kTimestamp: | |
| return VariantValueType::kTimestamp; | |
| case VariantDefs::kTimestampNtz: | |
| return VariantValueType::kTimestampNtz; | |
| case VariantDefs::kFloat: | |
| return VariantValueType::kFloat; | |
| case VariantDefs::kBinary: | |
| return VariantValueType::kBinary; | |
| case VariantDefs::kLongStr: | |
| return VariantValueType::kString; | |
| case VariantDefs::kUuid: | |
| return VariantValueType::kUuid; | |
| default: | |
| return UnknownPrimitiveTypeInVariant(type_info); | |
| } | |
| } | |
| } | |
| Result<int32_t> VariantBinaryUtil::ValueSize(std::string_view value, int32_t pos) { | |
| PAIMON_RETURN_NOT_OK(CheckIndex(pos, static_cast<int32_t>(value.size()))); | |
| auto header = static_cast<uint8_t>(value[pos]); | |
| int32_t basic_type = header & VariantDefs::kBasicTypeMask; | |
| int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; | |
| switch (basic_type) { | |
| case VariantDefs::kShortStr: | |
| return 1 + type_info; | |
| case VariantDefs::kObject: { | |
| PAIMON_ASSIGN_OR_RAISE(ObjectInfo info, GetObjectInfo(value, pos)); | |
| PAIMON_ASSIGN_OR_RAISE( | |
| int32_t data_size, | |
| ReadUnsigned(value, info.offset_start + info.num_elements * info.offset_size, | |
| info.offset_size)); | |
| return info.data_start - pos + data_size; | |
| } | |
| case VariantDefs::kArray: { | |
| PAIMON_ASSIGN_OR_RAISE(ArrayInfo info, GetArrayInfo(value, pos)); | |
| PAIMON_ASSIGN_OR_RAISE( | |
| int32_t data_size, | |
| ReadUnsigned(value, info.offset_start + info.num_elements * info.offset_size, | |
| info.offset_size)); | |
| return info.data_start - pos + data_size; | |
| } | |
| default: | |
| switch (type_info) { | |
| case VariantDefs::kNull: | |
| case VariantDefs::kTrue: | |
| case VariantDefs::kFalse: | |
| return 1; | |
| case VariantDefs::kInt1: | |
| return 2; | |
| case VariantDefs::kInt2: | |
| return 3; | |
| case VariantDefs::kInt4: | |
| case VariantDefs::kDate: | |
| case VariantDefs::kFloat: | |
| return 5; | |
| case VariantDefs::kInt8: | |
| case VariantDefs::kDouble: | |
| case VariantDefs::kTimestamp: | |
| case VariantDefs::kTimestampNtz: | |
| return 9; | |
| case VariantDefs::kDecimal4: | |
| return 6; | |
| case VariantDefs::kDecimal8: | |
| return 10; | |
| case VariantDefs::kDecimal16: | |
| return 18; | |
| case VariantDefs::kBinary: | |
| case VariantDefs::kLongStr: { | |
| PAIMON_ASSIGN_OR_RAISE(int32_t data_size, | |
| ReadUnsigned(value, pos + 1, VariantDefs::kU32Size)); | |
| return 1 + VariantDefs::kU32Size + data_size; | |
| } | |
| case VariantDefs::kUuid: | |
| return 17; | |
| default: | |
| return UnknownPrimitiveTypeInVariant(type_info); | |
| } | |
| } | |
| } | |
| Result<bool> VariantBinaryUtil::GetBoolean(std::string_view value, int32_t pos) { | |
| PAIMON_RETURN_NOT_OK(CheckIndex(pos, static_cast<int32_t>(value.size()))); | |
| auto header = static_cast<uint8_t>(value[pos]); | |
| int32_t basic_type = header & VariantDefs::kBasicTypeMask; | |
| int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; | |
| if (basic_type != VariantDefs::kPrimitive || | |
| (type_info != VariantDefs::kTrue && type_info != VariantDefs::kFalse)) { | |
| return UnexpectedType(VariantValueType::kBoolean); | |
| } | |
| return type_info == VariantDefs::kTrue; | |
| } | |
| Result<int64_t> VariantBinaryUtil::GetLong(std::string_view value, int32_t pos) { | |
| PAIMON_RETURN_NOT_OK(CheckIndex(pos, static_cast<int32_t>(value.size()))); | |
| auto header = static_cast<uint8_t>(value[pos]); | |
| int32_t basic_type = header & VariantDefs::kBasicTypeMask; | |
| int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; | |
| constexpr const char* kExceptionMessage = "Expect type to be LONG/DATE/TIMESTAMP/TIMESTAMP_NTZ"; | |
| if (basic_type != VariantDefs::kPrimitive) { | |
| return Status::Invalid(kExceptionMessage); | |
| } | |
| switch (type_info) { | |
| case VariantDefs::kInt1: | |
| return ReadLong(value, pos + 1, 1); | |
| case VariantDefs::kInt2: | |
| return ReadLong(value, pos + 1, 2); | |
| case VariantDefs::kInt4: | |
| case VariantDefs::kDate: | |
| return ReadLong(value, pos + 1, 4); | |
| case VariantDefs::kInt8: | |
| case VariantDefs::kTimestamp: | |
| case VariantDefs::kTimestampNtz: | |
| return ReadLong(value, pos + 1, 8); | |
| default: | |
| return Status::Invalid(kExceptionMessage); | |
| } | |
| } | |
| Result<double> VariantBinaryUtil::GetDouble(std::string_view value, int32_t pos) { | |
| PAIMON_RETURN_NOT_OK(CheckIndex(pos, static_cast<int32_t>(value.size()))); | |
| auto header = static_cast<uint8_t>(value[pos]); | |
| int32_t basic_type = header & VariantDefs::kBasicTypeMask; | |
| int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; | |
| if (basic_type != VariantDefs::kPrimitive || type_info != VariantDefs::kDouble) { | |
| return UnexpectedType(VariantValueType::kDouble); | |
| } | |
| PAIMON_ASSIGN_OR_RAISE(int64_t bits, ReadLong(value, pos + 1, 8)); | |
| double result; | |
| memcpy(&result, &bits, sizeof(result)); | |
| return result; | |
| } | |
| namespace { | |
| // Checks whether the precision and scale of the decimal are within the limit. | |
| Status CheckDecimal(const VariantDecimal& d, int32_t max_precision) { | |
| if (d.Precision() > max_precision || d.scale > max_precision) { | |
| return VariantBinaryUtil::MalformedVariant( | |
| fmt::format("decimal precision {} or scale {} exceeds the maximum precision {}", | |
| d.Precision(), d.scale, max_precision)); | |
| } | |
| return Status::OK(); | |
| } | |
| } // namespace | |
| Result<VariantDecimal> VariantBinaryUtil::GetDecimalWithOriginalScale(std::string_view value, | |
| int32_t pos) { | |
| auto length = static_cast<int32_t>(value.size()); | |
| PAIMON_RETURN_NOT_OK(CheckIndex(pos, length)); | |
| auto header = static_cast<uint8_t>(value[pos]); | |
| int32_t basic_type = header & VariantDefs::kBasicTypeMask; | |
| int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; | |
| if (basic_type != VariantDefs::kPrimitive) { | |
| return UnexpectedType(VariantValueType::kDecimal); | |
| } | |
| PAIMON_RETURN_NOT_OK(CheckIndex(pos + 1, length)); | |
| // Interpret the scale byte as unsigned. If it is a negative byte, the unsigned value must be | |
| // greater than `kMaxDecimal16Precision` and will trigger an error in `CheckDecimal`. | |
| int32_t scale = static_cast<uint8_t>(value[pos + 1]); | |
| VariantDecimal result; | |
| result.scale = scale; | |
| switch (type_info) { | |
| case VariantDefs::kDecimal4: { | |
| PAIMON_ASSIGN_OR_RAISE(int64_t unscaled, ReadLong(value, pos + 2, 4)); | |
| result.unscaled = unscaled; | |
| PAIMON_RETURN_NOT_OK(CheckDecimal(result, VariantDefs::kMaxDecimal4Precision)); | |
| break; | |
| } | |
| case VariantDefs::kDecimal8: { | |
| PAIMON_ASSIGN_OR_RAISE(int64_t unscaled, ReadLong(value, pos + 2, 8)); | |
| result.unscaled = unscaled; | |
| PAIMON_RETURN_NOT_OK(CheckDecimal(result, VariantDefs::kMaxDecimal8Precision)); | |
| break; | |
| } | |
| case VariantDefs::kDecimal16: { | |
| PAIMON_RETURN_NOT_OK(CheckIndex(pos + 17, length)); | |
| __uint128_t unscaled = 0; | |
| for (int32_t i = 0; i < 16; ++i) { | |
| unscaled |= static_cast<__uint128_t>(static_cast<uint8_t>(value[pos + 2 + i])) | |
| << (8 * i); | |
| } | |
| result.unscaled = static_cast<__int128_t>(unscaled); | |
| PAIMON_RETURN_NOT_OK(CheckDecimal(result, VariantDefs::kMaxDecimal16Precision)); | |
| break; | |
| } | |
| default: | |
| return UnexpectedType(VariantValueType::kDecimal); | |
| } | |
| return result; | |
| } | |
| Result<VariantDecimal> VariantBinaryUtil::GetDecimal(std::string_view value, int32_t pos) { | |
| PAIMON_ASSIGN_OR_RAISE(VariantDecimal decimal, GetDecimalWithOriginalScale(value, pos)); | |
| return decimal.StripTrailingZeros(); | |
| } | |
| Result<float> VariantBinaryUtil::GetFloat(std::string_view value, int32_t pos) { | |
| PAIMON_RETURN_NOT_OK(CheckIndex(pos, static_cast<int32_t>(value.size()))); | |
| auto header = static_cast<uint8_t>(value[pos]); | |
| int32_t basic_type = header & VariantDefs::kBasicTypeMask; | |
| int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; | |
| if (basic_type != VariantDefs::kPrimitive || type_info != VariantDefs::kFloat) { | |
| return UnexpectedType(VariantValueType::kFloat); | |
| } | |
| PAIMON_ASSIGN_OR_RAISE(int64_t bits, ReadLong(value, pos + 1, 4)); | |
| auto int_bits = static_cast<int32_t>(bits); | |
| float result; | |
| memcpy(&result, &int_bits, sizeof(result)); | |
| return result; | |
| } | |
| Result<std::string_view> VariantBinaryUtil::GetBinary(std::string_view value, int32_t pos) { | |
| auto length = static_cast<int32_t>(value.size()); | |
| PAIMON_RETURN_NOT_OK(CheckIndex(pos, length)); | |
| auto header = static_cast<uint8_t>(value[pos]); | |
| int32_t basic_type = header & VariantDefs::kBasicTypeMask; | |
| int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; | |
| if (basic_type != VariantDefs::kPrimitive || type_info != VariantDefs::kBinary) { | |
| return UnexpectedType(VariantValueType::kBinary); | |
| } | |
| int32_t start = pos + 1 + VariantDefs::kU32Size; | |
| PAIMON_ASSIGN_OR_RAISE(int32_t data_size, ReadUnsigned(value, pos + 1, VariantDefs::kU32Size)); | |
| if (data_size > 0) { | |
| PAIMON_RETURN_NOT_OK(CheckIndex(start + data_size - 1, length)); | |
| } | |
| return value.substr(start, data_size); | |
| } | |
| Result<std::string_view> VariantBinaryUtil::GetString(std::string_view value, int32_t pos) { | |
| auto length = static_cast<int32_t>(value.size()); | |
| PAIMON_RETURN_NOT_OK(CheckIndex(pos, length)); | |
| auto header = static_cast<uint8_t>(value[pos]); | |
| int32_t basic_type = header & VariantDefs::kBasicTypeMask; | |
| int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; | |
| if (basic_type == VariantDefs::kShortStr || | |
| (basic_type == VariantDefs::kPrimitive && type_info == VariantDefs::kLongStr)) { | |
| int32_t start; | |
| int32_t str_size; | |
| if (basic_type == VariantDefs::kShortStr) { | |
| start = pos + 1; | |
| str_size = type_info; | |
| } else { | |
| start = pos + 1 + VariantDefs::kU32Size; | |
| PAIMON_ASSIGN_OR_RAISE(str_size, ReadUnsigned(value, pos + 1, VariantDefs::kU32Size)); | |
| } | |
| if (str_size > 0) { | |
| PAIMON_RETURN_NOT_OK(CheckIndex(start + str_size - 1, length)); | |
| } | |
| return value.substr(start, str_size); | |
| } | |
| return UnexpectedType(VariantValueType::kString); | |
| } | |
| Result<std::string_view> VariantBinaryUtil::GetUuid(std::string_view value, int32_t pos) { | |
| auto length = static_cast<int32_t>(value.size()); | |
| PAIMON_RETURN_NOT_OK(CheckIndex(pos, length)); | |
| auto header = static_cast<uint8_t>(value[pos]); | |
| int32_t basic_type = header & VariantDefs::kBasicTypeMask; | |
| int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; | |
| if (basic_type != VariantDefs::kPrimitive || type_info != VariantDefs::kUuid) { | |
| return UnexpectedType(VariantValueType::kUuid); | |
| } | |
| int32_t start = pos + 1; | |
| PAIMON_RETURN_NOT_OK(CheckIndex(start + 15, length)); | |
| return value.substr(start, 16); | |
| } | |
| std::string VariantBinaryUtil::UuidToString(std::string_view uuid_bytes) { | |
| constexpr char kHexDigits[] = "0123456789abcdef"; | |
| std::string result; | |
| result.reserve(36); | |
| for (size_t i = 0; i < 16; ++i) { | |
| if (i == 4 || i == 6 || i == 8 || i == 10) { | |
| result.push_back('-'); | |
| } | |
| auto byte = static_cast<uint8_t>(uuid_bytes[i]); | |
| result.push_back(kHexDigits[byte >> 4]); | |
| result.push_back(kHexDigits[byte & 0xF]); | |
| } | |
| return result; | |
| } | |
| Result<VariantBinaryUtil::ObjectInfo> VariantBinaryUtil::GetObjectInfo(std::string_view value, | |
| int32_t pos) { | |
| PAIMON_RETURN_NOT_OK(CheckIndex(pos, static_cast<int32_t>(value.size()))); | |
| auto header = static_cast<uint8_t>(value[pos]); | |
| int32_t basic_type = header & VariantDefs::kBasicTypeMask; | |
| int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; | |
| if (basic_type != VariantDefs::kObject) { | |
| return UnexpectedType(VariantValueType::kObject); | |
| } | |
| // Refer to the comment of the `VariantDefs::kObject` constant for the details of the object | |
| // header encoding. Suppose `type_info` has a bit representation of 0_b4_b3b2_b1b0, the | |
| // following line extracts b4 to determine whether the object uses a 1/4-byte size. | |
| bool large_size = ((type_info >> 4) & 0x1) != 0; | |
| int32_t size_bytes = large_size ? VariantDefs::kU32Size : 1; | |
| ObjectInfo info; | |
| PAIMON_ASSIGN_OR_RAISE(info.num_elements, ReadUnsigned(value, pos + 1, size_bytes)); | |
| // Extracts b3b2 to determine the integer size of the field id list. | |
| info.id_size = ((type_info >> 2) & 0x3) + 1; | |
| // Extracts b1b0 to determine the integer size of the offset list. | |
| info.offset_size = (type_info & 0x3) + 1; | |
| info.id_start = pos + 1 + size_bytes; | |
| // A corrupted variant can claim a near-INT32_MAX element count; compute the layout in | |
| // 64-bit and bound it by the buffer before 32-bit arithmetic could overflow. | |
| int64_t offset_start = static_cast<int64_t>(info.id_start) + | |
| static_cast<int64_t>(info.num_elements) * info.id_size; | |
| int64_t data_start = | |
| offset_start + (static_cast<int64_t>(info.num_elements) + 1) * info.offset_size; | |
| if (data_start > static_cast<int64_t>(value.size())) { | |
| return MalformedVariant("object layout exceeds the value buffer"); | |
| } | |
| info.offset_start = static_cast<int32_t>(offset_start); | |
| info.data_start = static_cast<int32_t>(data_start); | |
| return info; | |
| } | |
| Result<VariantBinaryUtil::ArrayInfo> VariantBinaryUtil::GetArrayInfo(std::string_view value, | |
| int32_t pos) { | |
| PAIMON_RETURN_NOT_OK(CheckIndex(pos, static_cast<int32_t>(value.size()))); | |
| auto header = static_cast<uint8_t>(value[pos]); | |
| int32_t basic_type = header & VariantDefs::kBasicTypeMask; | |
| int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; | |
| if (basic_type != VariantDefs::kArray) { | |
| return UnexpectedType(VariantValueType::kArray); | |
| } | |
| // Suppose `type_info` has a bit representation of 000_b2_b1b0, the following line extracts b2 | |
| // to determine whether the array uses a 1/4-byte size. | |
| bool large_size = ((type_info >> 2) & 0x1) != 0; | |
| int32_t size_bytes = large_size ? VariantDefs::kU32Size : 1; | |
| ArrayInfo info; | |
| PAIMON_ASSIGN_OR_RAISE(info.num_elements, ReadUnsigned(value, pos + 1, size_bytes)); | |
| // Extracts b1b0 to determine the integer size of the offset list. | |
| info.offset_size = (type_info & 0x3) + 1; | |
| info.offset_start = pos + 1 + size_bytes; | |
| // See GetObjectInfo: bound the 64-bit layout by the buffer before 32-bit overflow. | |
| int64_t data_start = static_cast<int64_t>(info.offset_start) + | |
| (static_cast<int64_t>(info.num_elements) + 1) * info.offset_size; | |
| if (data_start > static_cast<int64_t>(value.size())) { | |
| return MalformedVariant("array layout exceeds the value buffer"); | |
| } | |
| info.data_start = static_cast<int32_t>(data_start); | |
| return info; | |
| } | |
| Result<std::string_view> VariantBinaryUtil::GetMetadataKey(std::string_view metadata, int32_t id) { | |
| auto length = static_cast<int32_t>(metadata.size()); | |
| PAIMON_RETURN_NOT_OK(CheckIndex(0, length)); | |
| // Extracts the highest 2 bits in the metadata header to determine the integer size of the | |
| // offset list. | |
| int32_t offset_size = ((static_cast<uint8_t>(metadata[0]) >> 6) & 0x3) + 1; | |
| PAIMON_ASSIGN_OR_RAISE(int32_t dict_size, ReadUnsigned(metadata, 1, offset_size)); | |
| if (id >= dict_size) { | |
| return MalformedVariant(fmt::format( | |
| "metadata key id {} is out of bounds for a dictionary of {} keys", id, dict_size)); | |
| } | |
| // There are a header byte, a `dict_size` with `offset_size` bytes, and `(dict_size + 1)` | |
| // offsets before the string data. | |
| // Bound the offset-table layout in 64-bit before 32-bit arithmetic could overflow on a | |
| // corrupted dictionary size. | |
| int64_t string_start64 = 1 + (static_cast<int64_t>(dict_size) + 2) * offset_size; | |
| if (string_start64 > static_cast<int64_t>(length)) { | |
| return MalformedVariant("metadata dictionary layout exceeds the metadata buffer"); | |
| } | |
| auto string_start = static_cast<int32_t>(string_start64); | |
| PAIMON_ASSIGN_OR_RAISE(int32_t offset, | |
| ReadUnsigned(metadata, 1 + (id + 1) * offset_size, offset_size)); | |
| PAIMON_ASSIGN_OR_RAISE(int32_t next_offset, | |
| ReadUnsigned(metadata, 1 + (id + 2) * offset_size, offset_size)); | |
| if (offset > next_offset) { | |
| return MalformedVariant("metadata key offsets are not monotonic"); | |
| } | |
| if (static_cast<int64_t>(string_start) + next_offset > static_cast<int64_t>(length)) { | |
| return MalformedVariant("metadata key data exceeds the metadata buffer"); | |
| } | |
| return metadata.substr(string_start + offset, next_offset - offset); | |
| } | |
| } // namespace paimon |