| // Licensed to the Apache Software Foundation (ASF) under one |
| // or more contributor license agreements. See the NOTICE file |
| // distributed with this work for additional information |
| // regarding copyright ownership. The ASF licenses this file |
| // to you under the Apache License, Version 2.0 (the |
| // "License"); you may not use this file except in compliance |
| // with the License. You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, |
| // software distributed under the License is distributed on an |
| // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| // KIND, either express or implied. See the License for the |
| // specific language governing permissions and limitations |
| // under the License. |
| |
| #include "olap/rowset/column_reader.h" |
| |
| #include <gtest/gtest.h> |
| |
| #include "olap/byte_buffer.h" |
| #include "olap/field.h" |
| #include "olap/olap_common.h" |
| #include "olap/olap_define.h" |
| #include "olap/row_block.h" |
| #include "olap/row_cursor.h" |
| #include "olap/rowset/column_writer.h" |
| #include "olap/stream_name.h" |
| #include "runtime/mem_pool.h" |
| #include "runtime/string_value.hpp" |
| #include "runtime/vectorized_row_batch.h" |
| #include "util/logging.h" |
| |
| using std::string; |
| |
| namespace doris { |
| |
| class TestColumn : public testing::Test { |
| public: |
| TestColumn() : _column_writer(NULL), _column_reader(NULL), _stream_factory(NULL) { |
| _offsets.clear(); |
| _mem_tracker.reset(new MemTracker(-1)); |
| _mem_pool.reset(new MemPool(_mem_tracker.get())); |
| } |
| |
| virtual ~TestColumn() { |
| SAFE_DELETE(_column_writer); |
| SAFE_DELETE(_column_reader); |
| SAFE_DELETE(_stream_factory); |
| } |
| |
| virtual void SetUp() { |
| _offsets.push_back(0); |
| |
| _stream_factory = new (std::nothrow) |
| OutStreamFactory(COMPRESS_LZ4, OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE); |
| ASSERT_TRUE(_stream_factory != NULL); |
| config::column_dictionary_key_ratio_threshold = 30; |
| config::column_dictionary_key_size_threshold = 1000; |
| } |
| |
| virtual void TearDown() { |
| SAFE_DELETE(_column_writer); |
| SAFE_DELETE(_column_reader); |
| SAFE_DELETE(_stream_factory); |
| SAFE_DELETE(_shared_buffer); |
| |
| _offsets.clear(); |
| for (auto in_stream : _map_in_streams) { |
| delete in_stream.second; |
| } |
| _map_in_streams.clear(); |
| _present_buffers.clear(); |
| _data_buffers.clear(); |
| _second_buffers.clear(); |
| _dictionary_buffers.clear(); |
| _length_buffers.clear(); |
| } |
| |
| void CreateColumnWriter(const TabletSchema& tablet_schema) { |
| _column_writer = ColumnWriter::create(0, tablet_schema, _stream_factory, 1024, |
| BLOOM_FILTER_DEFAULT_FPP); |
| |
| ASSERT_TRUE(_column_writer != NULL); |
| ASSERT_EQ(_column_writer->init(), OLAP_SUCCESS); |
| } |
| |
| void CreateColumnReader(const TabletSchema& tablet_schema) { |
| UniqueIdEncodingMap encodings; |
| encodings[0] = ColumnEncodingMessage(); |
| encodings[0].set_kind(ColumnEncodingMessage::DIRECT); |
| encodings[0].set_dictionary_size(1); |
| CreateColumnReader(tablet_schema, encodings); |
| } |
| |
| void CreateColumnReader(const TabletSchema& tablet_schema, UniqueIdEncodingMap& encodings) { |
| UniqueIdToColumnIdMap included; |
| included[0] = 0; |
| UniqueIdToColumnIdMap segment_included; |
| segment_included[0] = 0; |
| |
| _column_reader = |
| ColumnReader::create(0, tablet_schema, included, segment_included, encodings); |
| |
| ASSERT_TRUE(_column_reader != NULL); |
| |
| system("mkdir -p ./ut_dir"); |
| system("rm ./ut_dir/tmp_file"); |
| |
| ASSERT_EQ(OLAP_SUCCESS, |
| helper.open_with_mode("./ut_dir/tmp_file", O_CREAT | O_EXCL | O_WRONLY, |
| S_IRUSR | S_IWUSR)); |
| std::vector<int> off; |
| std::vector<int> length; |
| std::vector<int> buffer_size; |
| std::vector<StreamName> name; |
| |
| std::map<StreamName, OutStream*>::const_iterator it = _stream_factory->streams().begin(); |
| for (; it != _stream_factory->streams().end(); ++it) { |
| StreamName stream_name = it->first; |
| OutStream* out_stream = it->second; |
| std::vector<StorageByteBuffer*>* buffers; |
| |
| if (out_stream->is_suppressed()) { |
| continue; |
| } |
| |
| if (stream_name.kind() == StreamInfoMessage::ROW_INDEX) { |
| continue; |
| } else if (stream_name.kind() == StreamInfoMessage::PRESENT) { |
| buffers = &_present_buffers; |
| } else if (stream_name.kind() == StreamInfoMessage::DATA) { |
| buffers = &_data_buffers; |
| } else if (stream_name.kind() == StreamInfoMessage::SECONDARY) { |
| buffers = &_second_buffers; |
| } else if (stream_name.kind() == StreamInfoMessage::DICTIONARY_DATA) { |
| buffers = &_dictionary_buffers; |
| } else if (stream_name.kind() == StreamInfoMessage::LENGTH) { |
| buffers = &_length_buffers; |
| } else { |
| ASSERT_TRUE(false); |
| } |
| |
| ASSERT_TRUE(buffers != NULL); |
| off.push_back(helper.tell()); |
| out_stream->write_to_file(&helper, 0); |
| length.push_back(out_stream->get_stream_length()); |
| buffer_size.push_back(out_stream->get_total_buffer_size()); |
| name.push_back(stream_name); |
| } |
| helper.close(); |
| |
| ASSERT_EQ(OLAP_SUCCESS, |
| helper.open_with_mode("./ut_dir/tmp_file", O_RDONLY, S_IRUSR | S_IWUSR)); |
| |
| _shared_buffer = StorageByteBuffer::create(OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE + |
| sizeof(StreamHead)); |
| ASSERT_TRUE(_shared_buffer != NULL); |
| |
| for (int i = 0; i < off.size(); ++i) { |
| ReadOnlyFileStream* in_stream = new (std::nothrow) |
| ReadOnlyFileStream(&helper, &_shared_buffer, off[i], length[i], lz4_decompress, |
| buffer_size[i], &_stats); |
| ASSERT_EQ(OLAP_SUCCESS, in_stream->init()); |
| |
| _map_in_streams[name[i]] = in_stream; |
| } |
| |
| ASSERT_EQ(_column_reader->init(&_map_in_streams, 1024, _mem_pool.get(), &_stats), |
| OLAP_SUCCESS); |
| } |
| |
| void SetTabletSchemaWithOneColumn(std::string name, std::string type, std::string aggregation, |
| uint32_t length, bool is_allow_null, bool is_key, |
| TabletSchema* tablet_schema) { |
| TabletSchemaPB tablet_schema_pb; |
| ColumnPB* column = tablet_schema_pb.add_column(); |
| column->set_unique_id(0); |
| column->set_name(name); |
| column->set_type(type); |
| column->set_is_key(is_key); |
| column->set_is_nullable(is_allow_null); |
| column->set_length(length); |
| column->set_aggregation(aggregation); |
| column->set_precision(1000); |
| column->set_frac(1000); |
| column->set_is_bf_column(false); |
| tablet_schema->init_from_pb(tablet_schema_pb); |
| } |
| |
| void create_and_save_last_position() { |
| ASSERT_EQ(_column_writer->create_row_index_entry(), OLAP_SUCCESS); |
| } |
| |
| ColumnWriter* _column_writer; |
| ColumnReader* _column_reader; |
| std::shared_ptr<MemTracker> _mem_tracker; |
| std::unique_ptr<MemPool> _mem_pool; |
| std::unique_ptr<ColumnVector> _col_vector; |
| |
| OutStreamFactory* _stream_factory; |
| |
| std::vector<size_t> _offsets; |
| std::vector<StorageByteBuffer*> _present_buffers; |
| std::vector<StorageByteBuffer*> _data_buffers; |
| std::vector<StorageByteBuffer*> _second_buffers; |
| std::vector<StorageByteBuffer*> _dictionary_buffers; |
| std::vector<StorageByteBuffer*> _length_buffers; |
| |
| StorageByteBuffer* _shared_buffer; |
| std::map<StreamName, ReadOnlyFileStream*> _map_in_streams; |
| FileHandler helper; |
| OlapReaderStatistics _stats; |
| }; |
| |
| TEST_F(TestColumn, VectorizedTinyColumnWithoutPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| SetTabletSchemaWithOneColumn("TinyColumn", "TINYINT", "REPLACE", 1, false, true, |
| &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 1024; |
| block.init(block_info); |
| |
| char value = 1; |
| write_row.set_field_content(0, &value, _mem_pool.get()); |
| block.set_row(0, write_row); |
| |
| value = 3; |
| write_row.set_field_content(0, &value, _mem_pool.get()); |
| block.set_row(1, write_row); |
| block.finalize(2); |
| |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS); |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| |
| value = *reinterpret_cast<char*>(data); |
| ASSERT_EQ(value, 1); |
| |
| data++; |
| value = *reinterpret_cast<char*>(data); |
| ASSERT_EQ(value, 3); |
| } |
| |
| TEST_F(TestColumn, SeekTinyColumnWithoutPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| SetTabletSchemaWithOneColumn("TinyColumn", "TINYINT", "REPLACE", 1, false, true, |
| &tablet_schema); |
| |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 1024; |
| block.init(block_info); |
| |
| char value = 1; |
| write_row.set_field_content(0, &value, _mem_pool.get()); |
| block.set_row(0, write_row); |
| |
| value = 2; |
| write_row.set_field_content(0, &value, _mem_pool.get()); |
| block.set_row(1, write_row); |
| block.finalize(2); |
| |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| create_and_save_last_position(); |
| |
| value = 3; |
| write_row.set_field_content(0, &value, _mem_pool.get()); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| create_and_save_last_position(); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| PositionEntryReader entry1; |
| entry1._positions = _column_writer->index()->mutable_entry(0)->_positions; |
| entry1._positions_count = _column_writer->index()->mutable_entry(0)->_positions_count; |
| entry1._statistics.init(OLAP_FIELD_TYPE_TINYINT, false); |
| |
| PositionEntryReader entry2; |
| entry2._positions = _column_writer->index()->mutable_entry(1)->_positions; |
| entry2._positions_count = _column_writer->index()->mutable_entry(1)->_positions_count; |
| entry2._statistics.init(OLAP_FIELD_TYPE_TINYINT, false); |
| |
| PositionProvider position0(&entry1); |
| PositionProvider position1(&entry2); |
| |
| ASSERT_EQ(_column_reader->seek(&position0), OLAP_SUCCESS); |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS); |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| value = *reinterpret_cast<char*>(data); |
| ASSERT_EQ(value, 1); |
| data++; |
| value = *reinterpret_cast<char*>(data); |
| ASSERT_EQ(value, 2); |
| |
| ASSERT_EQ(_column_reader->seek(&position1), OLAP_SUCCESS); |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS); |
| data = reinterpret_cast<char*>(_col_vector->col_data()); |
| value = *reinterpret_cast<char*>(data); |
| ASSERT_EQ(value, 3); |
| } |
| |
| TEST_F(TestColumn, SkipTinyColumnWithoutPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| SetTabletSchemaWithOneColumn("TinyColumn", "TINYINT", "REPLACE", 1, false, true, |
| &tablet_schema); |
| |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 1024; |
| block.init(block_info); |
| |
| char value = 1; |
| write_row.set_field_content(0, &value, _mem_pool.get()); |
| block.set_row(0, write_row); |
| |
| value = 2; |
| write_row.set_field_content(0, &value, _mem_pool.get()); |
| block.set_row(1, write_row); |
| |
| value = 3; |
| write_row.set_field_content(0, &value, _mem_pool.get()); |
| block.set_row(2, write_row); |
| block.finalize(3); |
| |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| ASSERT_EQ(_column_reader->skip(2), OLAP_SUCCESS); |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS); |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| value = *reinterpret_cast<char*>(data); |
| ASSERT_EQ(value, 3); |
| } |
| |
| TEST_F(TestColumn, VectorizedTinyColumnWithPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| SetTabletSchemaWithOneColumn("TinyColumn", "TINYINT", "REPLACE", 1, true, true, &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 1024; |
| block.init(block_info); |
| |
| write_row.set_null(0); |
| block.set_row(0, write_row); |
| |
| write_row.set_not_null(0); |
| char value = 3; |
| write_row.set_field_content(0, &value, _mem_pool.get()); |
| block.set_row(1, write_row); |
| block.finalize(2); |
| |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS); |
| bool* is_null = _col_vector->is_null(); |
| ASSERT_EQ(is_null[0], true); |
| |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| ASSERT_EQ(is_null[1], false); |
| value = *reinterpret_cast<char*>(data + 1); |
| } |
| |
| TEST_F(TestColumn, TinyColumnIndex) { |
| // write data |
| TabletSchema tablet_schema; |
| SetTabletSchemaWithOneColumn("TinyColumn", "TINYINT", "REPLACE", 1, true, true, &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 1024; |
| block.init(block_info); |
| |
| char value = 1; |
| write_row.set_field_content(0, &value, _mem_pool.get()); |
| block.set_row(0, write_row); |
| |
| value = 3; |
| write_row.set_field_content(0, &value, _mem_pool.get()); |
| block.set_row(1, write_row); |
| block.finalize(2); |
| |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS); |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| value = *reinterpret_cast<char*>(data); |
| ASSERT_EQ(value, 1); |
| |
| value = *reinterpret_cast<char*>(data + 1); |
| ASSERT_EQ(value, 3); |
| } |
| |
| TEST_F(TestColumn, SeekTinyColumnWithPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| SetTabletSchemaWithOneColumn("TinyColumn", "TINYINT", "REPLACE", 1, true, true, &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 1024; |
| block.init(block_info); |
| |
| char value = 1; |
| write_row.set_field_content(0, &value, _mem_pool.get()); |
| block.set_row(0, write_row); |
| |
| value = 2; |
| write_row.set_field_content(0, &value, _mem_pool.get()); |
| block.set_row(1, write_row); |
| block.finalize(2); |
| |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| create_and_save_last_position(); |
| |
| value = 3; |
| write_row.set_field_content(0, &value, _mem_pool.get()); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| create_and_save_last_position(); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| PositionEntryReader entry1; |
| entry1._positions = _column_writer->index()->mutable_entry(0)->_positions; |
| entry1._positions_count = _column_writer->index()->mutable_entry(0)->_positions_count; |
| entry1._statistics.init(OLAP_FIELD_TYPE_TINYINT, false); |
| |
| PositionEntryReader entry2; |
| entry2._positions = _column_writer->index()->mutable_entry(1)->_positions; |
| entry2._positions_count = _column_writer->index()->mutable_entry(1)->_positions_count; |
| entry2._statistics.init(OLAP_FIELD_TYPE_TINYINT, false); |
| |
| PositionProvider position1(&entry1); |
| PositionProvider position2(&entry2); |
| |
| ASSERT_EQ(_column_reader->seek(&position1), OLAP_SUCCESS); |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS); |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| value = *reinterpret_cast<char*>(data); |
| ASSERT_EQ(value, 1); |
| value = *reinterpret_cast<char*>(data + 1); |
| ASSERT_EQ(value, 2); |
| |
| ASSERT_EQ(_column_reader->seek(&position2), OLAP_SUCCESS); |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS); |
| data = reinterpret_cast<char*>(_col_vector->col_data()); |
| value = *reinterpret_cast<char*>(data); |
| ASSERT_EQ(value, 3); |
| } |
| |
| TEST_F(TestColumn, SkipTinyColumnWithPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| SetTabletSchemaWithOneColumn("TinyColumn", "TINYINT", "REPLACE", 1, true, true, &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 1024; |
| block.init(block_info); |
| |
| char value = 1; |
| write_row.set_field_content(0, &value, _mem_pool.get()); |
| block.set_row(0, write_row); |
| |
| value = 2; |
| write_row.set_field_content(0, &value, _mem_pool.get()); |
| block.set_row(1, write_row); |
| |
| value = 3; |
| write_row.set_field_content(0, &value, _mem_pool.get()); |
| block.set_row(2, write_row); |
| block.finalize(3); |
| |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| ASSERT_EQ(_column_reader->skip(2), OLAP_SUCCESS); |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS); |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| value = *reinterpret_cast<char*>(data); |
| ASSERT_EQ(value, 3); |
| } |
| |
| TEST_F(TestColumn, VectorizedShortColumnWithoutPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| SetTabletSchemaWithOneColumn("ShortColumn", "SMALLINT", "REPLACE", 2, false, true, |
| &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 1024; |
| block.init(block_info); |
| |
| int16_t value = 1; |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(0, write_row); |
| |
| value = 3; |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(1, write_row); |
| block.finalize(2); |
| |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS); |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| value = *reinterpret_cast<int16_t*>(data); |
| ASSERT_EQ(value, 1); |
| |
| value = *reinterpret_cast<int16_t*>(data + sizeof(int16_t)); |
| ASSERT_EQ(value, 3); |
| } |
| |
| TEST_F(TestColumn, SeekShortColumnWithoutPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| SetTabletSchemaWithOneColumn("ShortColumn", "SMALLINT", "REPLACE", 2, false, true, |
| &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 1024; |
| block.init(block_info); |
| |
| int16_t value = 1; |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(0, write_row); |
| |
| value = 2; |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(1, write_row); |
| block.finalize(2); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| create_and_save_last_position(); |
| |
| value = 3; |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| create_and_save_last_position(); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| PositionEntryReader entry1; |
| entry1._positions = _column_writer->index()->mutable_entry(0)->_positions; |
| entry1._positions_count = _column_writer->index()->mutable_entry(0)->_positions_count; |
| entry1._statistics.init(OLAP_FIELD_TYPE_SMALLINT, false); |
| |
| PositionEntryReader entry2; |
| entry2._positions = _column_writer->index()->mutable_entry(1)->_positions; |
| entry2._positions_count = _column_writer->index()->mutable_entry(1)->_positions_count; |
| entry2._statistics.init(OLAP_FIELD_TYPE_SMALLINT, false); |
| |
| PositionProvider position0(&entry1); |
| PositionProvider position1(&entry2); |
| |
| ASSERT_EQ(_column_reader->seek(&position0), OLAP_SUCCESS); |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS); |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| value = *reinterpret_cast<char*>(data); |
| ASSERT_EQ(value, 1); |
| |
| value = *reinterpret_cast<char*>(data + sizeof(int16_t)); |
| ASSERT_EQ(value, 2); |
| |
| ASSERT_EQ(_column_reader->seek(&position1), OLAP_SUCCESS); |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS); |
| data = reinterpret_cast<char*>(_col_vector->col_data()); |
| value = *reinterpret_cast<char*>(data); |
| ASSERT_EQ(value, 3); |
| } |
| |
| TEST_F(TestColumn, SkipShortColumnWithoutPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| SetTabletSchemaWithOneColumn("ShortColumn", "SMALLINT", "REPLACE", 2, false, true, |
| &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 1024; |
| block.init(block_info); |
| |
| int16_t value = 1; |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(0, write_row); |
| |
| value = 2; |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(1, write_row); |
| |
| value = 3; |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(2, write_row); |
| block.finalize(3); |
| |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| ASSERT_EQ(_column_reader->skip(2), OLAP_SUCCESS); |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS); |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| value = *reinterpret_cast<char*>(data); |
| ASSERT_EQ(value, 3); |
| } |
| |
| TEST_F(TestColumn, SeekShortColumnWithPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| SetTabletSchemaWithOneColumn("ShortColumn", "SMALLINT", "REPLACE", 2, true, true, |
| &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 1024; |
| block.init(block_info); |
| |
| int16_t value = 1; |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(0, write_row); |
| |
| value = 2; |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(1, write_row); |
| block.finalize(2); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| create_and_save_last_position(); |
| |
| value = 3; |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| create_and_save_last_position(); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| PositionEntryReader entry1; |
| entry1._positions = _column_writer->index()->mutable_entry(0)->_positions; |
| entry1._positions_count = _column_writer->index()->mutable_entry(0)->_positions_count; |
| entry1._statistics.init(OLAP_FIELD_TYPE_SMALLINT, false); |
| |
| PositionEntryReader entry2; |
| entry2._positions = _column_writer->index()->mutable_entry(1)->_positions; |
| entry2._positions_count = _column_writer->index()->mutable_entry(1)->_positions_count; |
| entry2._statistics.init(OLAP_FIELD_TYPE_SMALLINT, false); |
| |
| PositionProvider position0(&entry1); |
| PositionProvider position1(&entry2); |
| |
| ASSERT_EQ(_column_reader->seek(&position0), OLAP_SUCCESS); |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS); |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| value = *reinterpret_cast<char*>(data); |
| ASSERT_EQ(value, 1); |
| |
| ASSERT_EQ(_column_reader->seek(&position1), OLAP_SUCCESS); |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS); |
| data = reinterpret_cast<char*>(_col_vector->col_data()); |
| value = *reinterpret_cast<char*>(data); |
| ASSERT_EQ(value, 3); |
| } |
| |
| TEST_F(TestColumn, VectorizedShortColumnWithPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| |
| SetTabletSchemaWithOneColumn("ShortColumn", "SMALLINT", "REPLACE", 2, true, true, |
| &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 1024; |
| block.init(block_info); |
| |
| write_row.set_null(0); |
| block.set_row(0, write_row); |
| |
| int16_t value = 3; |
| write_row.set_not_null(0); |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(1, write_row); |
| block.finalize(2); |
| |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS); |
| bool* is_null = _col_vector->is_null(); |
| ASSERT_EQ(is_null[0], true); |
| |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| ASSERT_EQ(is_null[1], false); |
| |
| value = *reinterpret_cast<char*>(data + sizeof(int16_t)); |
| ASSERT_EQ(value, 3); |
| } |
| |
| TEST_F(TestColumn, SkipShortColumnWithPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| SetTabletSchemaWithOneColumn("ShortColumn", "SMALLINT", "REPLACE", 2, true, true, |
| &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 1024; |
| block.init(block_info); |
| |
| int16_t value = 1; |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(0, write_row); |
| |
| value = 2; |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(1, write_row); |
| |
| value = 3; |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(2, write_row); |
| block.finalize(3); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| ASSERT_EQ(_column_reader->skip(2), OLAP_SUCCESS); |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS); |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| value = *reinterpret_cast<char*>(data); |
| ASSERT_EQ(value, 3); |
| } |
| |
| TEST_F(TestColumn, VectorizedIntColumnWithoutPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| SetTabletSchemaWithOneColumn("IntColumn", "INT", "REPLACE", 4, false, true, &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 1024; |
| block.init(block_info); |
| |
| int32_t value = 1; |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(0, write_row); |
| |
| value = 3; |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(1, write_row); |
| block.finalize(2); |
| |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS); |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| value = *reinterpret_cast<int*>(data); |
| ASSERT_EQ(value, 1); |
| |
| value = *reinterpret_cast<int*>(data + sizeof(int)); |
| ASSERT_EQ(value, 3); |
| } |
| |
| TEST_F(TestColumn, VectorizedIntColumnMassWithoutPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| SetTabletSchemaWithOneColumn("IntColumn", "INT", "REPLACE", 4, false, true, &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| for (int32_t i = 0; i < 10000; i++) { |
| write_row.set_field_content(0, reinterpret_cast<char*>(&i), _mem_pool.get()); |
| block.set_row(i, write_row); |
| } |
| block.finalize(10000); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| _col_vector.reset(new ColumnVector()); |
| |
| char* data = NULL; |
| for (int32_t i = 0; i < 10000; ++i) { |
| if (i % 1000 == 0) { |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 1000, _mem_pool.get()), |
| OLAP_SUCCESS); |
| data = reinterpret_cast<char*>(_col_vector->col_data()); |
| } |
| |
| int32_t value = 0; |
| value = *reinterpret_cast<int*>(data); |
| ASSERT_EQ(value, i); |
| data += sizeof(int32_t); |
| } |
| } |
| |
| TEST_F(TestColumn, VectorizedIntColumnWithPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| SetTabletSchemaWithOneColumn("IntColumn", "INT", "REPLACE", 4, true, true, &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| int32_t value = -1; |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| write_row.set_null(0); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| _col_vector.reset(new ColumnVector()); |
| |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS); |
| |
| bool* is_null = _col_vector->is_null(); |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| ASSERT_EQ(is_null[0], false); |
| value = *reinterpret_cast<int*>(data); |
| ASSERT_EQ(value, -1); |
| |
| ASSERT_EQ(is_null[1], true); |
| } |
| |
| TEST_F(TestColumn, VectorizedLongColumnWithoutPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| SetTabletSchemaWithOneColumn("LongColumnWithoutPresent", "BIGINT", "REPLACE", 8, false, true, |
| &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| int64_t value = 1; |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| value = 3; |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS); |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| value = *reinterpret_cast<int64_t*>(data); |
| ASSERT_EQ(value, 1); |
| |
| value = *reinterpret_cast<int64_t*>(data + sizeof(int64_t)); |
| ASSERT_EQ(value, 3); |
| } |
| |
| TEST_F(TestColumn, VectorizedLongColumnWithPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| SetTabletSchemaWithOneColumn("LongColumnWithPresent", "BIGINT", "REPLACE", 8, true, true, |
| &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| write_row.set_null(0); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| int64_t value = 3; |
| write_row.set_not_null(0); |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS); |
| bool* is_null = _col_vector->is_null(); |
| ASSERT_EQ(is_null[0], true); |
| |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| ASSERT_EQ(is_null[1], false); |
| |
| value = *reinterpret_cast<int64_t*>(data + sizeof(int64_t)); |
| ASSERT_EQ(value, 3); |
| } |
| |
| TEST_F(TestColumn, VectorizedFloatColumnWithoutPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| SetTabletSchemaWithOneColumn("FloatColumnWithoutPresent", "FLOAT", "REPLACE", 4, false, true, |
| &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| float value = 1.234; |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| value = 3.234; |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS); |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| value = *reinterpret_cast<float*>(data); |
| ASSERT_FLOAT_EQ(value, 1.234); |
| |
| data += sizeof(float); |
| value = *reinterpret_cast<float*>(data); |
| ASSERT_FLOAT_EQ(value, 3.234); |
| } |
| |
| TEST_F(TestColumn, VectorizedFloatColumnWithPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| |
| SetTabletSchemaWithOneColumn("FloatColumnWithPresent", "FLOAT", "REPLACE", 4, true, true, |
| &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| write_row.set_null(0); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| float value = 3.234; |
| write_row.set_not_null(0); |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS); |
| bool* is_null = _col_vector->is_null(); |
| ASSERT_EQ(is_null[0], true); |
| |
| ASSERT_EQ(is_null[1], false); |
| |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()) + sizeof(float); |
| value = *reinterpret_cast<float*>(data); |
| ASSERT_FLOAT_EQ(value, 3.234); |
| } |
| |
| TEST_F(TestColumn, SeekFloatColumnWithPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| |
| SetTabletSchemaWithOneColumn("FloatColumnWithPresent", "FLOAT", "REPLACE", 4, true, true, |
| &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| float value = 1.234; |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| create_and_save_last_position(); |
| |
| value = 3.234; |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| create_and_save_last_position(); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| PositionEntryReader entry1; |
| entry1._positions = _column_writer->index()->mutable_entry(0)->_positions; |
| entry1._positions_count = _column_writer->index()->mutable_entry(0)->_positions_count; |
| entry1._statistics.init(OLAP_FIELD_TYPE_FLOAT, false); |
| |
| PositionEntryReader entry2; |
| entry2._positions = _column_writer->index()->mutable_entry(1)->_positions; |
| entry2._positions_count = _column_writer->index()->mutable_entry(1)->_positions_count; |
| entry2._statistics.init(OLAP_FIELD_TYPE_FLOAT, false); |
| |
| PositionProvider position0(&entry1); |
| PositionProvider position1(&entry2); |
| |
| ASSERT_EQ(_column_reader->seek(&position0), OLAP_SUCCESS); |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS); |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| value = *reinterpret_cast<float*>(data); |
| ASSERT_FLOAT_EQ(value, 1.234); |
| |
| value = *reinterpret_cast<float*>(data + sizeof(float)); |
| ASSERT_FLOAT_EQ(value, 3.234); |
| } |
| |
| TEST_F(TestColumn, SkipFloatColumnWithPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| |
| SetTabletSchemaWithOneColumn("FloatColumnWithPresent", "FLOAT", "REPLACE", 4, true, true, |
| &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| float value = 1.234; |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| value = 3.234; |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| ASSERT_EQ(_column_reader->skip(1), OLAP_SUCCESS); |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS); |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| value = *reinterpret_cast<float*>(data); |
| ASSERT_FLOAT_EQ(value, 3.234); |
| } |
| |
| TEST_F(TestColumn, VectorizedDoubleColumnWithoutPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| |
| SetTabletSchemaWithOneColumn("DoubleColumnWithoutPresent", "DOUBLE", "REPLACE", 8, false, true, |
| &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| double value = 1.23456789; |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| value = 3.23456789; |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS); |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| value = *reinterpret_cast<double*>(data); |
| ASSERT_DOUBLE_EQ(value, 1.23456789); |
| |
| data += sizeof(double); |
| value = *reinterpret_cast<double*>(data); |
| ASSERT_DOUBLE_EQ(value, 3.23456789); |
| } |
| |
| TEST_F(TestColumn, VectorizedDoubleColumnWithPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| |
| SetTabletSchemaWithOneColumn("DoubleColumnWithPresent", "DOUBLE", "REPLACE", 8, true, true, |
| &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| write_row.set_null(0); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| double value = 3.23456789; |
| write_row.set_not_null(0); |
| write_row.set_field_content(0, reinterpret_cast<char*>(&value), _mem_pool.get()); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS); |
| bool* is_null = _col_vector->is_null(); |
| ASSERT_EQ(is_null[0], true); |
| |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| ASSERT_EQ(is_null[1], false); |
| |
| data += sizeof(double); |
| value = *reinterpret_cast<double*>(data); |
| ASSERT_DOUBLE_EQ(value, 3.23456789); |
| } |
| |
| TEST_F(TestColumn, VectorizedDatetimeColumnWithoutPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| |
| SetTabletSchemaWithOneColumn("DatetimeColumnWithoutPresent", "DATETIME", "REPLACE", 8, false, |
| true, &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| std::vector<string> val_string_array; |
| val_string_array.push_back("2000-10-10 10:10:10"); |
| OlapTuple tuple(val_string_array); |
| write_row.from_tuple(tuple); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS); |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| read_row.set_field_content(0, data, _mem_pool.get()); |
| ASSERT_TRUE(strncmp(read_row.to_string().c_str(), "0&2000-10-10 10:10:10", |
| strlen("0&2000-10-10 10:10:10")) == 0); |
| } |
| |
| TEST_F(TestColumn, VectorizedDatetimeColumnWithPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| |
| SetTabletSchemaWithOneColumn("DatetimeColumnWithoutPresent", "DATETIME", "REPLACE", 8, true, |
| true, &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| write_row.set_null(0); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| std::vector<string> val_string_array; |
| val_string_array.push_back("2000-10-10 10:10:10"); |
| OlapTuple tuple(val_string_array); |
| write_row.from_tuple(tuple); |
| write_row.set_not_null(0); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS); |
| bool* is_null = _col_vector->is_null(); |
| ASSERT_EQ(is_null[0], true); |
| |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| ASSERT_EQ(is_null[1], false); |
| |
| data += sizeof(uint64_t); |
| read_row.set_field_content(0, data, _mem_pool.get()); |
| ASSERT_TRUE(strncmp(read_row.to_string().c_str(), "0&2000-10-10 10:10:10", |
| strlen("0&2000-10-10 10:10:10")) == 0); |
| |
| ASSERT_NE(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS); |
| } |
| TEST_F(TestColumn, VectorizedDatetimeColumnZero) { |
| // write data |
| TabletSchema tablet_schema; |
| |
| SetTabletSchemaWithOneColumn("DatetimeColumnWithoutPresent", "DATETIME", "REPLACE", 8, true, |
| true, &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| write_row.set_null(0); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| std::vector<string> val_string_array; |
| val_string_array.push_back("1000-01-01 00:00:00"); |
| OlapTuple tuple(val_string_array); |
| write_row.from_tuple(tuple); |
| write_row.set_not_null(0); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS); |
| bool* is_null = _col_vector->is_null(); |
| ASSERT_EQ(is_null[0], true); |
| |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| ASSERT_EQ(is_null[1], false); |
| |
| data += sizeof(uint64_t); |
| read_row.set_field_content(0, data, _mem_pool.get()); |
| std::cout << read_row.to_string() << std::endl; |
| ASSERT_TRUE(strncmp(read_row.to_string().c_str(), "0&1000-01-01 00:00:00", |
| strlen("0&1000-01-01 00:00:00")) == 0); |
| |
| ASSERT_NE(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS); |
| } |
| |
| TEST_F(TestColumn, VectorizedDateColumnWithoutPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| |
| SetTabletSchemaWithOneColumn("DateColumnWithoutoutPresent", "DATE", "REPLACE", 3, false, true, |
| &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| std::vector<string> val_string_array; |
| val_string_array.push_back("2000-10-10"); |
| OlapTuple tuple(val_string_array); |
| write_row.from_tuple(tuple); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS); |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| read_row.set_field_content(0, data, _mem_pool.get()); |
| ASSERT_TRUE(strncmp(read_row.to_string().c_str(), "0&2000-10-10", strlen("0&2000-10-10")) == 0); |
| } |
| |
| TEST_F(TestColumn, VectorizedDateColumnWithPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| |
| SetTabletSchemaWithOneColumn("DateColumnWithoutoutPresent", "DATE", "REPLACE", 3, true, true, |
| &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| write_row.set_null(0); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| std::vector<string> val_string_array; |
| val_string_array.push_back("2000-10-10"); |
| OlapTuple tuple(val_string_array); |
| write_row.from_tuple(tuple); |
| for (uint32_t i = 0; i < 100; ++i) { |
| write_row.set_not_null(0); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| } |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 101, _mem_pool.get()), OLAP_SUCCESS); |
| bool* is_null = _col_vector->is_null(); |
| ASSERT_EQ(is_null[0], true); |
| |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| for (uint32_t i = 0; i < 100; ++i) { |
| data += sizeof(uint24_t); |
| ASSERT_EQ(is_null[i + 1], false); |
| read_row.set_field_content(0, data, _mem_pool.get()); |
| ASSERT_TRUE(strncmp(read_row.to_string().c_str(), "0&2000-10-10", strlen("0&2000-10-10")) == |
| 0); |
| } |
| } |
| |
| TEST_F(TestColumn, VectorizedDecimalColumnWithoutPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| |
| SetTabletSchemaWithOneColumn("DecimalColumnWithoutoutPresent", "DECIMAL", "REPLACE", 12, false, |
| true, &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| std::vector<string> val_string_array; |
| val_string_array.push_back("1234.5678"); |
| OlapTuple tuple1(val_string_array); |
| write_row.from_tuple(tuple1); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| val_string_array.clear(); |
| val_string_array.push_back("5678.1234"); |
| OlapTuple tuple2(val_string_array); |
| write_row.from_tuple(tuple2); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS); |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| read_row.set_field_content(0, data, _mem_pool.get()); |
| ASSERT_TRUE(strncmp(read_row.to_string().c_str(), "0&1234.5678", strlen("0&1234.5678")) == 0); |
| |
| data += sizeof(decimal12_t); |
| read_row.set_field_content(0, data, _mem_pool.get()); |
| ASSERT_TRUE(strncmp(read_row.to_string().c_str(), "0&5678.1234", strlen("0&5678.1234")) == 0); |
| } |
| |
| TEST_F(TestColumn, VectorizedDecimalColumnWithPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| |
| SetTabletSchemaWithOneColumn("DecimalColumnWithoutoutPresent", "DECIMAL", "REPLACE", 12, true, |
| true, &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| std::vector<string> val_string_array; |
| write_row.set_null(0); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| val_string_array.clear(); |
| val_string_array.push_back("5678.1234"); |
| OlapTuple tuple(val_string_array); |
| write_row.from_tuple(tuple); |
| write_row.set_not_null(0); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS); |
| bool* is_null = _col_vector->is_null(); |
| ASSERT_EQ(is_null[0], true); |
| |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| data += sizeof(decimal12_t); |
| ASSERT_EQ(is_null[1], false); |
| read_row.set_field_content(0, data, _mem_pool.get()); |
| ASSERT_TRUE(strncmp(read_row.to_string().c_str(), "0&5678.1234", strlen("0&5678.1234")) == 0); |
| } |
| |
| TEST_F(TestColumn, SkipDecimalColumnWithPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| |
| SetTabletSchemaWithOneColumn("DecimalColumnWithPresent", "DECIMAL", "REPLACE", 12, true, true, |
| &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| std::vector<string> val_string_array; |
| val_string_array.push_back("1234.5678"); |
| write_row.from_tuple(val_string_array); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| val_string_array.clear(); |
| val_string_array.push_back("5678.1234"); |
| write_row.from_tuple(val_string_array); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| char read_value[20]; |
| memset(read_value, 0, 20); |
| ASSERT_EQ(_column_reader->skip(1), OLAP_SUCCESS); |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS); |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| read_row.set_field_content(0, data, _mem_pool.get()); |
| ASSERT_TRUE(strncmp(read_row.to_string().c_str(), "0&5678.1234", strlen("0&5678.1234")) == 0); |
| } |
| |
| TEST_F(TestColumn, SeekDecimalColumnWithPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| SetTabletSchemaWithOneColumn("DecimalColumnWithPresent", "DECIMAL", "REPLACE", 12, true, true, |
| &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| std::vector<string> val_string_array; |
| val_string_array.push_back("1234.5678"); |
| OlapTuple tuple1(val_string_array); |
| write_row.from_tuple(tuple1); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| create_and_save_last_position(); |
| |
| val_string_array.clear(); |
| val_string_array.push_back("5678.1234"); |
| OlapTuple tuple2(val_string_array); |
| write_row.from_tuple(tuple2); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| create_and_save_last_position(); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| PositionEntryReader entry1; |
| entry1._positions = _column_writer->index()->mutable_entry(0)->_positions; |
| entry1._positions_count = _column_writer->index()->mutable_entry(0)->_positions_count; |
| entry1._statistics.init(OLAP_FIELD_TYPE_FLOAT, false); |
| |
| PositionEntryReader entry2; |
| entry2._positions = _column_writer->index()->mutable_entry(1)->_positions; |
| entry2._positions_count = _column_writer->index()->mutable_entry(1)->_positions_count; |
| entry2._statistics.init(OLAP_FIELD_TYPE_FLOAT, false); |
| |
| PositionProvider position0(&entry1); |
| PositionProvider position1(&entry2); |
| |
| char read_value[20]; |
| memset(read_value, 0, 20); |
| ASSERT_EQ(_column_reader->seek(&position0), OLAP_SUCCESS); |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS); |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| read_row.set_field_content(0, data, _mem_pool.get()); |
| ASSERT_TRUE(strncmp(read_row.to_string().c_str(), "0&1234.5678", strlen("0&1234.5678")) == 0); |
| |
| memset(read_value, 0, 20); |
| ASSERT_EQ(_column_reader->seek(&position1), OLAP_SUCCESS); |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS); |
| data = reinterpret_cast<char*>(_col_vector->col_data()); |
| read_row.set_field_content(0, data, _mem_pool.get()); |
| ASSERT_TRUE(strncmp(read_row.to_string().c_str(), "0&5678.1234", strlen("0&5678.1234")) == 0); |
| } |
| |
| TEST_F(TestColumn, VectorizedLargeIntColumnWithoutPresent) { |
| // init tablet schema |
| TabletSchema tablet_schema; |
| |
| SetTabletSchemaWithOneColumn("LargeIntColumnWithoutoutPresent", "LARGEINT", "SUM", 16, false, |
| true, &tablet_schema); |
| // test data |
| string value1 = "100000000000000000000000000000000000000"; |
| string value2 = "-170141183460469231731687303715884105728"; |
| |
| // write data |
| CreateColumnWriter(tablet_schema); |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| std::vector<string> val_string_array; |
| val_string_array.push_back(value1); |
| OlapTuple tuple1(val_string_array); |
| write_row.from_tuple(tuple1); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| val_string_array.clear(); |
| val_string_array.push_back(value2); |
| OlapTuple tuple2(val_string_array); |
| write_row.from_tuple(tuple2); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS); |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| read_row.set_field_content(0, data, _mem_pool.get()); |
| value1 = "0&" + value1; |
| value2 = "0&" + value2; |
| ASSERT_TRUE(strncmp(read_row.to_string().c_str(), value1.c_str(), value1.size()) == 0); |
| |
| read_row.set_field_content(0, data + sizeof(int128_t), _mem_pool.get()); |
| ASSERT_TRUE(strncmp(read_row.to_string().c_str(), value2.c_str(), value2.size()) == 0); |
| } |
| |
| TEST_F(TestColumn, VectorizedLargeIntColumnWithPresent) { |
| // init tablet schema |
| TabletSchema tablet_schema; |
| |
| SetTabletSchemaWithOneColumn("LargeIntColumnWithoutoutPresent", "LARGEINT", "SUM", 16, true, |
| true, &tablet_schema); |
| |
| // test data |
| string value1 = "100000000000000000000000000000000000000"; |
| string value2 = "-170141183460469231731687303715884105728"; |
| |
| // write data |
| CreateColumnWriter(tablet_schema); |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| write_row.set_null(0); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| std::vector<string> val_string_array; |
| val_string_array.push_back(value1); |
| OlapTuple tuple1(val_string_array); |
| write_row.from_tuple(tuple1); |
| write_row.set_not_null(0); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| val_string_array.clear(); |
| val_string_array.push_back(value2); |
| OlapTuple tuple2(val_string_array); |
| write_row.from_tuple(tuple2); |
| write_row.set_not_null(0); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 3, _mem_pool.get()), OLAP_SUCCESS); |
| bool* is_null = _col_vector->is_null(); |
| ASSERT_EQ(is_null[0], true); |
| ASSERT_EQ(is_null[1], false); |
| ASSERT_EQ(is_null[2], false); |
| |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| value1 = "0&" + value1; |
| value2 = "0&" + value2; |
| |
| data += sizeof(int128_t); |
| read_row.set_field_content(0, data, _mem_pool.get()); |
| ASSERT_TRUE(strncmp(read_row.to_string().c_str(), value1.c_str(), value1.size()) == 0); |
| |
| data += sizeof(int128_t); |
| read_row.set_field_content(0, data, _mem_pool.get()); |
| ASSERT_TRUE(strncmp(read_row.to_string().c_str(), value2.c_str(), value2.size()) == 0); |
| } |
| |
| TEST_F(TestColumn, SkipLargeIntColumnWithPresent) { |
| // init tablet schema |
| TabletSchema tablet_schema; |
| |
| SetTabletSchemaWithOneColumn("LargeIntColumnWithPresent", "LARGEINT", "SUM", 16, true, true, |
| &tablet_schema); |
| // test data |
| string value1 = "100000000000000000000000000000000000000"; |
| string value2 = "-170141183460469231731687303715884105728"; |
| |
| // write data |
| CreateColumnWriter(tablet_schema); |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| std::vector<string> val_string_array; |
| val_string_array.push_back(value1); |
| OlapTuple tuple1(val_string_array); |
| write_row.from_tuple(tuple1); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| val_string_array.clear(); |
| val_string_array.push_back(value2); |
| OlapTuple tuple2(val_string_array); |
| write_row.from_tuple(tuple2); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| |
| value2 = "0&" + value2; |
| ASSERT_EQ(_column_reader->skip(1), OLAP_SUCCESS); |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS); |
| char* data = reinterpret_cast<char*>(_col_vector->col_data()); |
| read_row.set_field_content(0, data, _mem_pool.get()); |
| ASSERT_TRUE(strncmp(read_row.to_string().c_str(), value2.c_str(), value2.size()) == 0); |
| } |
| |
| TEST_F(TestColumn, VectorizedDirectVarcharColumnWithoutPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| |
| SetTabletSchemaWithOneColumn("DirectVarcharColumnWithoutoutPresent", "VARCHAR", "REPLACE", 10, |
| false, true, &tablet_schema); |
| |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| write_row.allocate_memory_for_string_type(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| std::vector<string> val_string_array; |
| val_string_array.push_back("YWJjZGU="); //"abcde" base_64_encode is "YWJjZGU=" |
| OlapTuple tuple1(val_string_array); |
| write_row.from_tuple(tuple1); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| for (uint32_t i = 0; i < 2; i++) { |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| } |
| val_string_array.clear(); |
| val_string_array.push_back("ZWRjYmE="); //"edcba" base_64_encode is "ZWRjYmE=" |
| OlapTuple tuple2(val_string_array); |
| write_row.from_tuple(tuple2); |
| for (uint32_t i = 0; i < 2; i++) { |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| } |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| read_row.allocate_memory_for_string_type(tablet_schema); |
| |
| _col_vector.reset(new ColumnVector()); |
| |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 5, _mem_pool.get()), OLAP_SUCCESS); |
| Slice* value = reinterpret_cast<Slice*>(_col_vector->col_data()); |
| ASSERT_TRUE(strncmp(value->data, "YWJjZGU=", value->size) == 0); |
| for (uint32_t i = 0; i < 2; i++) { |
| value++; |
| ASSERT_TRUE(strncmp(value->data, "YWJjZGU=", value->size) == 0); |
| } |
| for (uint32_t i = 0; i < 2; i++) { |
| value++; |
| ASSERT_TRUE(strncmp(value->data, "ZWRjYmE=", value->size) == 0); |
| } |
| ASSERT_NE(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS); |
| } |
| |
| TEST_F(TestColumn, VectorizedDirectVarcharColumnWithPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| SetTabletSchemaWithOneColumn("DirectVarcharColumnWithoutoutPresent", "VARCHAR", "REPLACE", 10, |
| true, true, &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| write_row.allocate_memory_for_string_type(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| write_row.set_null(0); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| std::vector<string> val_string_array; |
| val_string_array.push_back("YWJjZGU="); //"abcde" base_64_encode is "YWJjZGU=" |
| OlapTuple tuple(val_string_array); |
| write_row.from_tuple(tuple); |
| write_row.set_not_null(0); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| read_row.allocate_memory_for_string_type(tablet_schema); |
| |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS); |
| bool* is_null = _col_vector->is_null(); |
| ASSERT_EQ(is_null[0], true); |
| ASSERT_EQ(is_null[1], false); |
| |
| Slice* value = reinterpret_cast<Slice*>(_col_vector->col_data()); |
| value++; |
| ASSERT_TRUE(strncmp(value->data, "YWJjZGU=", value->size) == 0); |
| } |
| |
| TEST_F(TestColumn, SkipDirectVarcharColumnWithPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| |
| SetTabletSchemaWithOneColumn("DirectVarcharColumnWithPresent", "VARCHAR", "REPLACE", 10, true, |
| true, &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| write_row.allocate_memory_for_string_type(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| std::vector<string> val_string_array; |
| val_string_array.push_back("YWJjZGU="); //"abcde" base_64_encode is "YWJjZGU=" |
| OlapTuple tuple1(val_string_array); |
| write_row.from_tuple(tuple1); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| val_string_array.clear(); |
| val_string_array.push_back("YWFhYWE="); //"aaaaa" base_64_encode is "YWJjZGU=" |
| OlapTuple tuple2(val_string_array); |
| write_row.from_tuple(tuple2); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| read_row.allocate_memory_for_string_type(tablet_schema); |
| |
| char read_value[20]; |
| memset(read_value, 0, 20); |
| ASSERT_EQ(_column_reader->skip(1), OLAP_SUCCESS); |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS); |
| Slice* value = reinterpret_cast<Slice*>(_col_vector->col_data()); |
| ASSERT_TRUE(strncmp(value->data, "YWFhYWE=", value->size) == 0); |
| } |
| |
| TEST_F(TestColumn, SeekDirectVarcharColumnWithoutPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| |
| SetTabletSchemaWithOneColumn("DirectVarcharColumnWithPresent", "VARCHAR", "REPLACE", 10, false, |
| true, &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| write_row.allocate_memory_for_string_type(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| std::vector<string> val_string_array; |
| val_string_array.push_back("YWJjZGU="); //"abcde" base_64_encode is "YWJjZGU=" |
| OlapTuple tuple1(val_string_array); |
| write_row.from_tuple(tuple1); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| _column_writer->create_row_index_entry(); |
| |
| val_string_array.clear(); |
| val_string_array.push_back("YWFhYWE="); //"aaaaa" base_64_encode is "YWJjZGU=" |
| OlapTuple tuple2(val_string_array); |
| write_row.from_tuple(tuple2); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| _column_writer->create_row_index_entry(); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| read_row.allocate_memory_for_string_type(tablet_schema); |
| |
| PositionEntryReader entry1; |
| entry1._positions = _column_writer->index()->mutable_entry(0)->_positions; |
| entry1._positions_count = _column_writer->index()->mutable_entry(0)->_positions_count; |
| entry1._statistics.init(OLAP_FIELD_TYPE_VARCHAR, false); |
| |
| PositionEntryReader entry2; |
| entry2._positions = _column_writer->index()->mutable_entry(1)->_positions; |
| entry2._positions_count = _column_writer->index()->mutable_entry(1)->_positions_count; |
| entry2._statistics.init(OLAP_FIELD_TYPE_VARCHAR, false); |
| |
| PositionProvider position0(&entry1); |
| PositionProvider position1(&entry2); |
| |
| ASSERT_EQ(_column_reader->seek(&position0), OLAP_SUCCESS); |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS); |
| Slice* value = reinterpret_cast<Slice*>(_col_vector->col_data()); |
| ASSERT_TRUE(strncmp(value->data, "YWJjZGU=", value->size) == 0); |
| |
| ASSERT_EQ(_column_reader->seek(&position1), OLAP_SUCCESS); |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS); |
| value = reinterpret_cast<Slice*>(_col_vector->col_data()); |
| ASSERT_TRUE(strncmp(value->data, "YWFhYWE=", value->size) == 0); |
| } |
| |
| TEST_F(TestColumn, SeekDirectVarcharColumnWithPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| |
| SetTabletSchemaWithOneColumn("DirectVarcharColumnWithPresent", "VARCHAR", "REPLACE", 10, true, |
| true, &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| write_row.allocate_memory_for_string_type(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| std::vector<string> val_string_array; |
| val_string_array.push_back("YWJjZGU="); //"abcde" base_64_encode is "YWJjZGU=" |
| OlapTuple tuple1(val_string_array); |
| write_row.from_tuple(tuple1); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| _column_writer->create_row_index_entry(); |
| |
| val_string_array.clear(); |
| val_string_array.push_back("YWFhYWE="); //"aaaaa" base_64_encode is "YWJjZGU=" |
| OlapTuple tuple2(val_string_array); |
| write_row.from_tuple(tuple2); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| _column_writer->create_row_index_entry(); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| read_row.allocate_memory_for_string_type(tablet_schema); |
| |
| PositionEntryReader entry1; |
| entry1._positions = _column_writer->index()->mutable_entry(0)->_positions; |
| entry1._positions_count = _column_writer->index()->mutable_entry(0)->_positions_count; |
| entry1._statistics.init(OLAP_FIELD_TYPE_VARCHAR, false); |
| |
| PositionEntryReader entry2; |
| entry2._positions = _column_writer->index()->mutable_entry(1)->_positions; |
| entry2._positions_count = _column_writer->index()->mutable_entry(1)->_positions_count; |
| entry2._statistics.init(OLAP_FIELD_TYPE_VARCHAR, false); |
| |
| PositionProvider position0(&entry1); |
| PositionProvider position1(&entry2); |
| |
| ASSERT_EQ(_column_reader->seek(&position0), OLAP_SUCCESS); |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS); |
| Slice* value = reinterpret_cast<Slice*>(_col_vector->col_data()); |
| ASSERT_TRUE(strncmp(value->data, "YWJjZGU=", value->size) == 0); |
| |
| ASSERT_EQ(_column_reader->seek(&position1), OLAP_SUCCESS); |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS); |
| value = reinterpret_cast<Slice*>(_col_vector->col_data()); |
| ASSERT_TRUE(strncmp(value->data, "YWFhYWE=", value->size) == 0); |
| } |
| |
| TEST_F(TestColumn, VectorizedStringColumnWithoutPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| SetTabletSchemaWithOneColumn("VarcharColumnWithoutoutPresent", "CHAR", "REPLACE", |
| strlen("abcde"), false, true, &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| write_row.allocate_memory_for_string_type(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| std::vector<string> val_string_array; |
| val_string_array.push_back("abcde"); //"abcde" base_64_encode is "YWJjZGU=" |
| OlapTuple tuple1(val_string_array); |
| write_row.from_tuple(tuple1); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| for (uint32_t i = 0; i < 2; i++) { |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| } |
| val_string_array.clear(); |
| val_string_array.push_back("edcba"); //"edcba" base_64_encode is "ZWRjYmE=" |
| OlapTuple tuple2(val_string_array); |
| write_row.from_tuple(tuple2); |
| for (uint32_t i = 0; i < 2; i++) { |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| } |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| read_row.allocate_memory_for_string_type(tablet_schema); |
| |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 5, _mem_pool.get()), OLAP_SUCCESS); |
| Slice* value = reinterpret_cast<Slice*>(_col_vector->col_data()); |
| |
| ASSERT_TRUE(strncmp(value->data, "abcde", value->size) == 0); |
| for (uint32_t i = 0; i < 2; i++) { |
| value++; |
| ASSERT_TRUE(strncmp(value->data, "abcde", value->size) == 0); |
| } |
| for (uint32_t i = 0; i < 2; i++) { |
| value++; |
| ASSERT_TRUE(strncmp(value->data, "edcba", value->size) == 0); |
| } |
| ASSERT_NE(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS); |
| } |
| |
| TEST_F(TestColumn, VectorizedStringColumnWithPresent) { |
| // write data |
| TabletSchema tablet_schema; |
| SetTabletSchemaWithOneColumn("VarcharColumnWithoutoutPresent", "CHAR", "REPLACE", |
| strlen("abcde"), true, true, &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| write_row.allocate_memory_for_string_type(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| write_row.set_null(0); |
| block.set_row(0, write_row); |
| |
| std::vector<string> val_string_array; |
| val_string_array.push_back("abcde"); //"abcde" base_64_encode is "YWJjZGU=" |
| OlapTuple tuple(val_string_array); |
| write_row.from_tuple(tuple); |
| write_row.set_not_null(0); |
| block.set_row(1, write_row); |
| block.finalize(2); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| read_row.allocate_memory_for_string_type(tablet_schema); |
| |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS); |
| bool* is_null = _col_vector->is_null(); |
| ASSERT_EQ(is_null[0], true); |
| ASSERT_EQ(is_null[1], false); |
| |
| Slice* value = reinterpret_cast<Slice*>(_col_vector->col_data()); |
| value++; |
| ASSERT_TRUE(strncmp(value->data, "abcde", value->size) == 0); |
| } |
| |
| TEST_F(TestColumn, VectorizedStringColumnWithoutoutPresent2) { |
| // write data |
| TabletSchema tablet_schema; |
| SetTabletSchemaWithOneColumn("VarcharColumnWithoutoutPresent", "CHAR", "REPLACE", 20, false, |
| true, &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| write_row.allocate_memory_for_string_type(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| std::vector<string> val_string_array; |
| val_string_array.push_back("abcde"); //"abcde" base_64_encode is "YWJjZGU=" |
| OlapTuple tuple1(val_string_array); |
| write_row.from_tuple(tuple1); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| val_string_array.clear(); |
| val_string_array.push_back("aaaaa"); //"abcde" base_64_encode is "YWJjZGU=" |
| OlapTuple tuple2(val_string_array); |
| write_row.from_tuple(tuple2); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| val_string_array.clear(); |
| val_string_array.push_back("bbbbb"); //"abcde" base_64_encode is "YWJjZGU=" |
| OlapTuple tuple3(val_string_array); |
| write_row.from_tuple(tuple3); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| val_string_array.clear(); |
| val_string_array.push_back("ccccc"); //"abcde" base_64_encode is "YWJjZGU=" |
| OlapTuple tuple4(val_string_array); |
| write_row.from_tuple(tuple4); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| val_string_array.clear(); |
| val_string_array.push_back("ddddd"); //"abcde" base_64_encode is "YWJjZGU=" |
| OlapTuple tuple5(val_string_array); |
| write_row.from_tuple(tuple5); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| read_row.allocate_memory_for_string_type(tablet_schema); |
| |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 5, _mem_pool.get()), OLAP_SUCCESS); |
| Slice* value = reinterpret_cast<Slice*>(_col_vector->col_data()); |
| |
| ASSERT_TRUE(strncmp(value->data, "abcde", value->size) == 0); |
| |
| value++; |
| ASSERT_TRUE(strncmp(value->data, "aaaaa", value->size) == 0); |
| |
| value++; |
| ASSERT_TRUE(strncmp(value->data, "bbbbb", value->size) == 0); |
| |
| value++; |
| ASSERT_TRUE(strncmp(value->data, "ccccc", value->size) == 0); |
| |
| value++; |
| ASSERT_TRUE(strncmp(value->data, "ddddd", value->size) == 0); |
| } |
| |
| TEST_F(TestColumn, VectorizedDirectVarcharColumnWith65533) { |
| // write data |
| TabletSchema tablet_schema; |
| SetTabletSchemaWithOneColumn("DirectVarcharColumnWithoutoutPresent", "VARCHAR", "REPLACE", |
| 65535, false, true, &tablet_schema); |
| CreateColumnWriter(tablet_schema); |
| |
| RowCursor write_row; |
| write_row.init(tablet_schema); |
| write_row.allocate_memory_for_string_type(tablet_schema); |
| |
| RowBlock block(&tablet_schema); |
| RowBlockInfo block_info; |
| block_info.row_num = 10000; |
| block.init(block_info); |
| |
| std::vector<string> val_string_array; |
| val_string_array.push_back(std::string(65533, 'a')); |
| OlapTuple tuple1(val_string_array); |
| ASSERT_EQ(OLAP_SUCCESS, write_row.from_tuple(tuple1)); |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| |
| val_string_array.clear(); |
| val_string_array.push_back("edcba"); //"edcba" base_64_encode is "ZWRjYmE=" |
| OlapTuple tuple2(val_string_array); |
| write_row.from_tuple(tuple2); |
| for (uint32_t i = 0; i < 2; i++) { |
| block.set_row(0, write_row); |
| block.finalize(1); |
| ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); |
| } |
| |
| ColumnDataHeaderMessage header; |
| ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); |
| |
| // read data |
| CreateColumnReader(tablet_schema); |
| |
| RowCursor read_row; |
| read_row.init(tablet_schema); |
| read_row.allocate_memory_for_string_type(tablet_schema); |
| |
| _col_vector.reset(new ColumnVector()); |
| ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 3, _mem_pool.get()), OLAP_SUCCESS); |
| Slice* value = reinterpret_cast<Slice*>(_col_vector->col_data()); |
| |
| for (uint32_t i = 0; i < 65533; i++) { |
| ASSERT_TRUE(strncmp(value->data + i, "a", 1) == 0); |
| } |
| |
| for (uint32_t i = 0; i < 2; i++) { |
| value++; |
| ASSERT_TRUE(strncmp(value->data, "edcba", value->size) == 0); |
| } |
| } |
| |
| } // namespace doris |
| |
| int main(int argc, char** argv) { |
| std::string conffile = std::string(getenv("DORIS_HOME")) + "/conf/be.conf"; |
| if (!doris::config::init(conffile.c_str(), false)) { |
| fprintf(stderr, "error read config file. \n"); |
| return -1; |
| } |
| int ret = doris::OLAP_SUCCESS; |
| testing::InitGoogleTest(&argc, argv); |
| ret = RUN_ALL_TESTS(); |
| return ret; |
| } |
