blob: 9ef118c10d3542447da3c046663fba36cc02f6ca [file]
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include "agent/be_exec_version_manager.h"
#include "core/column/column_array.h"
#include "core/column/column_fixed_length_object.h"
#include "core/column/column_nullable.h"
#include "core/data_type/data_type_agg_state.h"
#include "core/data_type/data_type_nullable.h"
#include "core/data_type/data_type_number.h"
#include "exprs/aggregate/aggregate_function_state_combine.h"
#include "exprs/aggregate/aggregate_function_state_merge.h"
#include "exprs/aggregate/aggregate_function_state_union.h"
#include "testutil/column_helper.h"
namespace doris {
class AggregateStateCombineTest : public testing::Test {
protected:
static DataTypePtr create_avg_state_type(const DataTypePtr& argument_type) {
return std::make_shared<DataTypeAggState>(DataTypes {argument_type}, true, "avg",
BeExecVersionManager::get_newest_version());
}
static ColumnPtr combine(AggregateFunctionPtr combine_function, const DataTypePtr& state_type,
const ColumnPtr& input_column, Arena& arena) {
auto* place =
reinterpret_cast<AggregateDataPtr>(arena.alloc(combine_function->size_of_data()));
combine_function->create(place);
const IColumn* columns[] = {input_column.get()};
combine_function->add_batch_single_place(input_column->size(), place, columns, arena);
auto result = state_type->create_column();
combine_function->insert_result_into(place, *result);
combine_function->destroy(place);
return result;
}
static ColumnPtr merge(AggregateFunctionPtr nested_function, const DataTypePtr& state_type,
const ColumnPtr& state_column, Arena& arena) {
auto merge_function = AggregateStateMerge::create(nested_function, DataTypes {state_type},
nested_function->get_return_type());
auto* place =
reinterpret_cast<AggregateDataPtr>(arena.alloc(merge_function->size_of_data()));
merge_function->create(place);
const IColumn* columns[] = {state_column.get()};
merge_function->add_batch_single_place(state_column->size(), place, columns, arena);
auto result = nested_function->get_return_type()->create_column();
merge_function->insert_result_into(place, *result);
merge_function->destroy(place);
return result;
}
};
TEST_F(AggregateStateCombineTest, AvgStateCompatibleWithStateUnion) {
auto argument_type = std::make_shared<DataTypeInt64>();
auto state_type = create_avg_state_type(argument_type);
auto nested_function =
assert_cast<const DataTypeAggState*>(state_type.get())->get_nested_function();
auto combine_function =
AggregateStateCombine::create(nested_function, DataTypes {argument_type}, state_type);
auto input_column = ColumnHelper::create_column<DataTypeInt64>({1, 2, 3});
Arena arena;
auto combined_state = combine(combine_function, state_type, input_column, arena);
auto partial_states = combine_function->create_serialize_column();
EXPECT_NE(check_and_get_column<ColumnFixedLengthObject>(*partial_states), nullptr);
EXPECT_TRUE(combine_function->get_serialized_type()->equals(
*nested_function->get_serialized_type()));
const IColumn* partial_input_columns[] = {input_column.get()};
combine_function->streaming_agg_serialize_to_column(partial_input_columns, partial_states,
input_column->size(), arena);
auto* partial_place =
reinterpret_cast<AggregateDataPtr>(arena.alloc(combine_function->size_of_data()));
combine_function->create(partial_place);
combine_function->deserialize_and_merge_from_column_range(partial_place, *partial_states, 0,
partial_states->size() - 1, arena);
auto partial_state = state_type->create_column();
combine_function->insert_result_into(partial_place, *partial_state);
auto partial_state_without_key = state_type->create_column();
combine_function->serialize_without_key_to_column(partial_place, *partial_state_without_key);
combine_function->destroy(partial_place);
auto row_states = nested_function->create_serialize_column();
const IColumn* input_columns[] = {input_column.get()};
nested_function->streaming_agg_serialize_to_column(input_columns, row_states,
input_column->size(), arena);
auto union_function =
AggregateStateUnion::create(nested_function, DataTypes {state_type}, state_type);
auto* union_place =
reinterpret_cast<AggregateDataPtr>(arena.alloc(union_function->size_of_data()));
union_function->create(union_place);
const IColumn* state_columns[] = {row_states.get()};
union_function->add_batch_single_place(row_states->size(), union_place, state_columns, arena);
auto union_state = state_type->create_column();
union_function->insert_result_into(union_place, *union_state);
union_function->destroy(union_place);
ASSERT_EQ(combined_state->size(), 1);
ASSERT_EQ(partial_state->size(), 1);
ASSERT_EQ(partial_state_without_key->size(), 1);
ASSERT_EQ(union_state->size(), 1);
EXPECT_EQ(combined_state->get_data_at(0), partial_state->get_data_at(0));
EXPECT_EQ(combined_state->get_data_at(0), partial_state_without_key->get_data_at(0));
EXPECT_EQ(combined_state->get_data_at(0), union_state->get_data_at(0));
auto result = merge(nested_function, state_type, combined_state, arena);
const auto& result_column = assert_cast<const ColumnFloat64&>(*result);
ASSERT_EQ(result_column.size(), 1);
EXPECT_DOUBLE_EQ(result_column.get_data()[0], 2.0);
}
TEST_F(AggregateStateCombineTest, AvgNullableInput) {
auto argument_type = make_nullable(std::make_shared<DataTypeInt64>());
auto state_type = create_avg_state_type(argument_type);
auto nested_function =
assert_cast<const DataTypeAggState*>(state_type.get())->get_nested_function();
auto combine_function =
AggregateStateCombine::create(nested_function, DataTypes {argument_type}, state_type);
auto input_column = ColumnHelper::create_nullable_column<DataTypeInt64>({1, 0, 3}, {0, 1, 0});
Arena arena;
auto combined_state = combine(combine_function, state_type, input_column, arena);
auto result = merge(nested_function, state_type, combined_state, arena);
const auto& nullable_result = assert_cast<const ColumnNullable&>(*result);
ASSERT_EQ(nullable_result.size(), 1);
ASSERT_FALSE(nullable_result.is_null_at(0));
const auto& result_column =
assert_cast<const ColumnFloat64&>(nullable_result.get_nested_column());
EXPECT_DOUBLE_EQ(result_column.get_data()[0], 2.0);
auto all_null_input = ColumnHelper::create_nullable_column<DataTypeInt64>({0, 0}, {1, 1});
auto all_null_state = combine(combine_function, state_type, all_null_input, arena);
auto all_null_result = merge(nested_function, state_type, all_null_state, arena);
const auto& nullable_all_null_result = assert_cast<const ColumnNullable&>(*all_null_result);
ASSERT_EQ(nullable_all_null_result.size(), 1);
EXPECT_TRUE(nullable_all_null_result.is_null_at(0));
}
TEST_F(AggregateStateCombineTest, CountNullableInputPreservesGroupedStates) {
auto argument_type = make_nullable(std::make_shared<DataTypeInt64>());
auto state_type = std::make_shared<DataTypeAggState>(
DataTypes {argument_type}, false, "count", BeExecVersionManager::get_newest_version());
auto nested_function = state_type->get_nested_function();
auto combine_function =
AggregateStateCombine::create(nested_function, DataTypes {argument_type}, state_type);
auto input_column =
ColumnHelper::create_nullable_column<DataTypeInt64>({10, 0, 20, 30}, {0, 1, 0, 0});
Arena arena;
std::vector<AggregateDataPtr> combine_places(2);
for (auto& place : combine_places) {
place = reinterpret_cast<AggregateDataPtr>(arena.alloc(combine_function->size_of_data()));
combine_function->create(place);
}
AggregateDataPtr row_places[] = {combine_places[0], combine_places[0], combine_places[1],
combine_places[1]};
const IColumn* input_columns[] = {input_column.get()};
combine_function->add_batch(input_column->size(), row_places, 0, input_columns, arena, false);
auto combined_states = state_type->create_column();
combine_function->insert_result_into_vec(combine_places, 0, *combined_states,
combine_places.size());
ASSERT_EQ(combined_states->size(), 2);
combine_function->insert_result_into(combine_places[0], *combined_states);
ASSERT_EQ(combined_states->size(), 3);
auto merge_function = AggregateStateMerge::create(nested_function, DataTypes {state_type},
nested_function->get_return_type());
std::vector<AggregateDataPtr> merge_places(combined_states->size());
for (auto& place : merge_places) {
place = reinterpret_cast<AggregateDataPtr>(arena.alloc(merge_function->size_of_data()));
merge_function->create(place);
}
const IColumn* state_columns[] = {combined_states.get()};
merge_function->add_batch(combined_states->size(), merge_places.data(), 0, state_columns, arena,
false);
auto result = nested_function->get_return_type()->create_column();
merge_function->insert_result_into_vec(merge_places, 0, *result, merge_places.size());
const auto& count_result = assert_cast<const ColumnInt64&>(*result);
ASSERT_EQ(count_result.size(), 3);
EXPECT_EQ(count_result.get_data()[0], 1);
EXPECT_EQ(count_result.get_data()[1], 2);
EXPECT_EQ(count_result.get_data()[2], 1);
for (auto* place : combine_places) {
combine_function->destroy(place);
}
for (auto* place : merge_places) {
merge_function->destroy(place);
}
}
TEST_F(AggregateStateCombineTest, LargeGroupedArrayState) {
constexpr size_t group_count = 8;
constexpr size_t rows_per_group = 4096;
constexpr size_t row_count = group_count * rows_per_group;
auto argument_type = std::make_shared<DataTypeInt64>();
auto state_type =
std::make_shared<DataTypeAggState>(DataTypes {argument_type}, false, "array_agg",
BeExecVersionManager::get_newest_version());
auto nested_function = state_type->get_nested_function();
auto combine_function =
AggregateStateCombine::create(nested_function, DataTypes {argument_type}, state_type);
std::vector<Int64> values(row_count);
std::vector<AggregateDataPtr> row_places(row_count);
std::vector<AggregateDataPtr> combine_places(group_count);
Arena arena;
for (size_t group = 0; group < group_count; ++group) {
combine_places[group] =
reinterpret_cast<AggregateDataPtr>(arena.alloc(combine_function->size_of_data()));
combine_function->create(combine_places[group]);
for (size_t row = 0; row < rows_per_group; ++row) {
const size_t index = group * rows_per_group + row;
values[index] = static_cast<Int64>(index);
row_places[index] = combine_places[group];
}
}
auto input_column = ColumnHelper::create_column<DataTypeInt64>(values);
const IColumn* input_columns[] = {input_column.get()};
combine_function->add_batch(row_count, row_places.data(), 0, input_columns, arena, false);
auto combined_states = state_type->create_column();
combine_function->insert_result_into_vec(combine_places, 0, *combined_states, group_count);
ASSERT_EQ(combined_states->size(), group_count);
auto merge_function = AggregateStateMerge::create(nested_function, DataTypes {state_type},
nested_function->get_return_type());
std::vector<AggregateDataPtr> merge_places(group_count);
for (auto& place : merge_places) {
place = reinterpret_cast<AggregateDataPtr>(arena.alloc(merge_function->size_of_data()));
merge_function->create(place);
}
const IColumn* state_columns[] = {combined_states.get()};
merge_function->add_batch(group_count, merge_places.data(), 0, state_columns, arena, false);
auto result = nested_function->get_return_type()->create_column();
merge_function->insert_result_into_vec(merge_places, 0, *result, group_count);
const auto& array_result = assert_cast<const ColumnArray&>(*result);
ASSERT_EQ(array_result.size(), group_count);
ASSERT_EQ(array_result.get_data().size(), row_count);
for (size_t group = 0; group < group_count; ++group) {
EXPECT_EQ(array_result.get_offsets()[group], (group + 1) * rows_per_group);
}
for (auto* place : combine_places) {
combine_function->destroy(place);
}
for (auto* place : merge_places) {
merge_function->destroy(place);
}
}
} // namespace doris