be/test/exprs/function/function_hash_test.cpp

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

#include "exprs/function/function_hash.h"

#include <stdint.h>

#include <string>
#include <vector>

#include "common/status.h"
#include "core/data_type/data_type_nullable.h"
#include "core/data_type/data_type_number.h"
#include "core/data_type/data_type_string.h"
#include "core/types.h"
#include "core/value/large_int_value.h"
#include "exprs/function/function_test_util.h"
#include "gtest/gtest_pred_impl.h"
#include "testutil/any_type.h"
#include "util/hash/murmur_hash3.h"

namespace doris {
using namespace ut_type;

namespace {

__int128_t pack_murmur_hash3_128_for_test(uint64_t h1, uint64_t h2) {
    static_assert(sizeof(__int128_t) == sizeof(uint64_t) * 2);
    const auto value =
            (static_cast<unsigned __int128>(h2) << 64) | static_cast<unsigned __int128>(h1);
    return static_cast<__int128_t>(value);
}

__int128_t murmur_hash3_128_for_test(const std::vector<std::string>& values) {
    uint64_t h1 = 0;
    uint64_t h2 = 0;
    for (const std::string& value : values) {
        murmur_hash3_x64_process(value.data(), value.size(), h1, h2);
    }
    return pack_murmur_hash3_128_for_test(h1, h2);
}

std::string murmur_hash3_u128_for_test(const std::vector<std::string>& values) {
    return LargeIntValue::to_string(static_cast<__uint128_t>(murmur_hash3_128_for_test(values)));
}

} // namespace

TEST(HashFunctionTest, murmur_hash_3_test) {
    std::string func_name = "murmur_hash3_32";

    {
        InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR};

        DataSet data_set = {{{Null()}, Null()}, {{std::string("hello")}, (int32_t)1321743225}};

        static_cast<void>(check_function<DataTypeInt32, true>(func_name, input_types, data_set));
    };

    {
        InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR};

        DataSet data_set = {{{std::string("hello"), std::string("world")}, (int32_t)984713481},
                            {{std::string("hello"), Null()}, Null()}};

        static_cast<void>(check_function<DataTypeInt32, true>(func_name, input_types, data_set));
    };

    {
        InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR,
                                    PrimitiveType::TYPE_VARCHAR};

        DataSet data_set = {{{std::string("hello"), std::string("world"), std::string("!")},
                             (int32_t)-666935433},
                            {{std::string("hello"), std::string("world"), Null()}, Null()}};

        static_cast<void>(check_function<DataTypeInt32, true>(func_name, input_types, data_set));
    };
}

TEST(HashFunctionTest, murmur_hash_3_64_test) {
    std::string func_name = "murmur_hash3_64";

    {
        InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR};

        DataSet data_set = {{{Null()}, Null()},
                            {{std::string("hello")}, (int64_t)-3215607508166160593}};

        static_cast<void>(check_function<DataTypeInt64, true>(func_name, input_types, data_set));
    };

    {
        InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR};

        DataSet data_set = {
                {{std::string("hello"), std::string("world")}, (int64_t)3583109472027628045},
                {{std::string("hello"), Null()}, Null()}};

        static_cast<void>(check_function<DataTypeInt64, true>(func_name, input_types, data_set));
    };

    {
        InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR,
                                    PrimitiveType::TYPE_VARCHAR};

        DataSet data_set = {{{std::string("hello"), std::string("world"), std::string("!")},
                             (int64_t)1887828212617890932},
                            {{std::string("hello"), std::string("world"), Null()}, Null()}};

        static_cast<void>(check_function<DataTypeInt64, true>(func_name, input_types, data_set));
    };
}

TEST(HashFunctionTest, murmur_hash_3_64_v2_test) {
    std::string func_name = "murmur_hash3_64_v2";

    {
        InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR};

        DataSet data_set = {{{std::string("1000209601_1756808272")}, (int64_t)4038800892574899471},
                            {{std::string("hello world")}, (int64_t)5998619086395760910},
                            {{std::string("apache doris")}, (int64_t)3669213779466221743}};

        static_cast<void>(check_function<DataTypeInt64, true>(func_name, input_types, data_set));
    };
}

TEST(HashFunctionTest, murmur_hash_3_128_test) {
    std::string func_name = "murmur_hash3_128";

    {
        InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR};

        DataSet data_set = {
                {{Null()}, Null()},
                {{std::string("hello world")},
                 pack_murmur_hash3_128_for_test(5998619086395760910ULL, 12364428806279881649ULL)}};

        static_cast<void>(check_function<DataTypeInt128, true>(func_name, input_types, data_set));
        check_function_all_arg_comb<DataTypeInt128, true>(func_name, input_types, data_set);
    };

    {
        InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR};

        DataSet data_set = {{{std::string("hello"), std::string("world")},
                             murmur_hash3_128_for_test({"hello", "world"})},
                            {{std::string("hello"), Null()}, Null()}};

        static_cast<void>(check_function<DataTypeInt128, true>(func_name, input_types, data_set));
        check_function_all_arg_comb<DataTypeInt128, true>(func_name, input_types, data_set);
    };

    {
        InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR,
                                    PrimitiveType::TYPE_VARCHAR};

        DataSet data_set = {{{std::string("hello"), std::string("world"), std::string("!")},
                             murmur_hash3_128_for_test({"hello", "world", "!"})},
                            {{std::string("hello"), std::string("world"), Null()}, Null()}};

        check_function_all_arg_comb<DataTypeInt128, true>(func_name, input_types, data_set);
    };
}

TEST(HashFunctionTest, murmur_hash_3_128_empty_arguments_test) {
    auto check_empty_arguments = [](const std::string& func_name, const DataTypePtr& return_type) {
        Block block;
        FunctionBasePtr func = SimpleFunctionFactory::instance().get_function(
                func_name, block.get_columns_with_type_and_name(), return_type);
        ASSERT_TRUE(func != nullptr);

        FunctionUtils fn_utils(return_type, {}, false);
        auto* fn_ctx = fn_utils.get_fn_ctx();
        ASSERT_TRUE(func->open(fn_ctx, FunctionContext::FRAGMENT_LOCAL).ok());
        ASSERT_TRUE(func->open(fn_ctx, FunctionContext::THREAD_LOCAL).ok());

        block.insert({nullptr, return_type, "result"});
        const auto st = func->execute(fn_ctx, block, {}, 0, 1);
        EXPECT_FALSE(st.ok());
        EXPECT_NE(st.to_string().find("requires at least one argument"), std::string::npos);

        static_cast<void>(func->close(fn_ctx, FunctionContext::THREAD_LOCAL));
        static_cast<void>(func->close(fn_ctx, FunctionContext::FRAGMENT_LOCAL));
    };

    check_empty_arguments("murmur_hash3_128", std::make_shared<DataTypeInt128>());
    check_empty_arguments("murmur_hash3_u128", std::make_shared<DataTypeString>());
}

TEST(HashFunctionTest, murmur_hash_3_u128_test) {
    std::string func_name = "murmur_hash3_u128";

    {
        InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR};

        DataSet data_set = {
                {{Null()}, Null()},
                {{std::string("")}, std::string("0")},
                {{std::string("hello world")}, murmur_hash3_u128_for_test({"hello world"})}};

        static_cast<void>(check_function<DataTypeString, true>(func_name, input_types, data_set));
        check_function_all_arg_comb<DataTypeString, true>(func_name, input_types, data_set);
    };

    {
        InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR};

        DataSet data_set = {{{std::string("hello"), std::string("world")},
                             murmur_hash3_u128_for_test({"hello", "world"})},
                            {{std::string("hello"), Null()}, Null()}};

        static_cast<void>(check_function<DataTypeString, true>(func_name, input_types, data_set));
        check_function_all_arg_comb<DataTypeString, true>(func_name, input_types, data_set);
    };

    {
        InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR,
                                    PrimitiveType::TYPE_VARCHAR};

        DataSet data_set = {{{std::string("hello"), std::string("world"), std::string("!")},
                             murmur_hash3_u128_for_test({"hello", "world", "!"})},
                            {{std::string("hello"), std::string("world"), Null()}, Null()}};

        check_function_all_arg_comb<DataTypeString, true>(func_name, input_types, data_set);
    };
}

TEST(HashFunctionTest, murmur_hash_3_u128_unsigned_range_test) {
    std::string func_name = "murmur_hash3_u128";

    InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR};
    DataSet data_set = {
            {{std::string("hello world")}, std::string("228083453807047072434243676435732455694")}};

    static_cast<void>(check_function<DataTypeString, true>(func_name, input_types, data_set));
}

TEST(HashFunctionTest, murmur_hash_get_name_test) {
    EXPECT_STREQ(murmur_hash3_get_name_type_int_for_test(), "murmur_hash3_32");
    EXPECT_STREQ(murmur_hash3_get_name_type_bigint_for_test(), "murmur_hash3_64");
    EXPECT_STREQ(murmur_hash3_get_name_type_bigint_v2_for_test(), "murmur_hash3_64_v2");
}

TEST(HashFunctionTest, xxhash_32_test) {
    std::string func_name = "xxhash_32";

    {
        InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR};

        DataSet data_set = {{{Null()}, Null()}, {{std::string("hello")}, (int32_t)-83855367}};

        static_cast<void>(check_function<DataTypeInt32, true>(func_name, input_types, data_set));
    };

    {
        InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR};

        DataSet data_set = {{{std::string("hello"), std::string("world")}, (int32_t)-920844969},
                            {{std::string("hello"), Null()}, Null()}};

        static_cast<void>(check_function<DataTypeInt32, true>(func_name, input_types, data_set));
    };

    {
        InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR,
                                    PrimitiveType::TYPE_VARCHAR};

        DataSet data_set = {{{std::string("hello"), std::string("world"), std::string("!")},
                             (int32_t)352087701},
                            {{std::string("hello"), std::string("world"), Null()}, Null()}};

        static_cast<void>(check_function<DataTypeInt32, true>(func_name, input_types, data_set));
    };

    {
        InputTypeSet input_types = {PrimitiveType::TYPE_VARBINARY};

        DataSet data_set = {{{Null()}, Null()}, {{VARBINARY("hello")}, (int32_t)-83855367}};

        static_cast<void>(check_function<DataTypeInt32, true>(func_name, input_types, data_set));
    };

    {
        InputTypeSet input_types = {PrimitiveType::TYPE_VARBINARY, PrimitiveType::TYPE_VARBINARY};

        DataSet data_set = {{{VARBINARY("hello"), VARBINARY("world")}, (int32_t)-920844969},
                            {{VARBINARY("hello"), Null()}, Null()}};

        static_cast<void>(check_function<DataTypeInt32, true>(func_name, input_types, data_set));
    };

    {
        InputTypeSet input_types = {PrimitiveType::TYPE_VARBINARY, PrimitiveType::TYPE_VARBINARY,
                                    PrimitiveType::TYPE_VARBINARY};

        DataSet data_set = {
                {{VARBINARY("hello"), VARBINARY("world"), VARBINARY("!")}, (int32_t)352087701},
                {{VARBINARY("hello"), VARBINARY("world"), Null()}, Null()}};

        static_cast<void>(check_function<DataTypeInt32, true>(func_name, input_types, data_set));
    };
}

TEST(HashFunctionTest, xxhash_64_test) {
    std::string func_name = "xxhash_64";

    {
        InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR};

        DataSet data_set = {{{Null()}, Null()},
                            {{std::string("hello")}, (int64_t)-7685981735718036227}};

        static_cast<void>(check_function<DataTypeInt64, true>(func_name, input_types, data_set));
    };

    {
        InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR};

        DataSet data_set = {
                {{std::string("hello"), std::string("world")}, (int64_t)7001965798170371843},
                {{std::string("hello"), Null()}, Null()}};

        static_cast<void>(check_function<DataTypeInt64, true>(func_name, input_types, data_set));
    };

    {
        InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR,
                                    PrimitiveType::TYPE_VARCHAR};

        DataSet data_set = {{{std::string("hello"), std::string("world"), std::string("!")},
                             (int64_t)6796829678999971400},
                            {{std::string("hello"), std::string("world"), Null()}, Null()}};

        static_cast<void>(check_function<DataTypeInt64, true>(func_name, input_types, data_set));
    };

    {
        InputTypeSet input_types = {PrimitiveType::TYPE_VARBINARY};

        DataSet data_set = {{{Null()}, Null()},
                            {{VARBINARY("hello")}, (int64_t)-7685981735718036227}};

        static_cast<void>(check_function<DataTypeInt64, true>(func_name, input_types, data_set));
    };

    {
        InputTypeSet input_types = {PrimitiveType::TYPE_VARBINARY, PrimitiveType::TYPE_VARBINARY};

        DataSet data_set = {
                {{VARBINARY("hello"), VARBINARY("world")}, (int64_t)7001965798170371843},
                {{VARBINARY("hello"), Null()}, Null()}};

        static_cast<void>(check_function<DataTypeInt64, true>(func_name, input_types, data_set));
    };

    {
        InputTypeSet input_types = {PrimitiveType::TYPE_VARBINARY, PrimitiveType::TYPE_VARBINARY,
                                    PrimitiveType::TYPE_VARBINARY};

        DataSet data_set = {{{VARBINARY("hello"), VARBINARY("world"), VARBINARY("!")},
                             (int64_t)6796829678999971400},
                            {{VARBINARY("hello"), VARBINARY("world"), Null()}, Null()}};

        static_cast<void>(check_function<DataTypeInt64, true>(func_name, input_types, data_set));
    };
}

TEST(HashFunctionTest, murmur_hash3_helper_functions_test) {
    {
        std::string input = "hello world";
        uint64_t h1 = 0;
        uint64_t h2 = 0;
        murmur_hash3_x64_process(input.data(), input.size(), h1, h2);
        EXPECT_EQ(h1, 5998619086395760910ULL);
        EXPECT_EQ(h2, 12364428806279881649ULL);
    }

    {
        std::string input = "hello world";
        uint64_t out[2] = {0, 0};
        murmur_hash3_x64_128(input.data(), input.size(), 0, out);
        EXPECT_TRUE(out[0] == 5998619086395760910ULL && out[1] == 12364428806279881649ULL);
    }

    {
        std::string input = "hello world";
        uint64_t out = 0;
        murmur_hash3_x64_64_shared(input.data(), input.size(), 0, &out);
        EXPECT_EQ(out, 5998619086395760910ULL);
    }

    {
        std::string input = "hello";
        uint64_t out = 0;
        murmur_hash3_x64_64(input.data(), input.size(), 0, &out);
        EXPECT_EQ(out, static_cast<uint64_t>(-3215607508166160593LL));
    }

    {
        std::string input = "";
        uint64_t h1 = 0, h2 = 0;
        murmur_hash3_x64_process(input.data(), input.size(), h1, h2);
        EXPECT_EQ(h1, 0ULL);
        EXPECT_EQ(h2, 0ULL);
    }
}

} // namespace doris