| // Licensed to the Apache Software Foundation (ASF) under one |
| // or more contributor license agreements. See the NOTICE file |
| // distributed with this work for additional information |
| // regarding copyright ownership. The ASF licenses this file |
| // to you under the Apache License, Version 2.0 (the |
| // "License"); you may not use this file except in compliance |
| // with the License. You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, |
| // software distributed under the License is distributed on an |
| // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| // KIND, either express or implied. See the License for the |
| // specific language governing permissions and limitations |
| // under the License. |
| |
| #include "storage/index/index_file_reader.h" |
| |
| #include <memory> |
| #include <utility> |
| |
| #include "common/cast_set.h" |
| #include "common/config.h" |
| #include "storage/index/inverted/inverted_index_compound_reader.h" |
| #include "storage/index/inverted/inverted_index_fs_directory.h" |
| #include "storage/tablet/tablet_schema.h" |
| #include "util/debug_points.h" |
| |
| namespace doris::segment_v2 { |
| |
| Status IndexFileReader::init(int32_t read_buffer_size, const io::IOContext* io_ctx) { |
| std::unique_lock<std::shared_mutex> lock(_mutex); // Lock for writing |
| if (!_inited) { |
| _read_buffer_size = read_buffer_size; |
| if (_storage_format == InvertedIndexStorageFormatPB::SNII) { |
| RETURN_IF_ERROR(_init_snii(io_ctx)); |
| } else if (_storage_format >= InvertedIndexStorageFormatPB::V2) { |
| RETURN_IF_ERROR(_init_from(read_buffer_size, io_ctx)); |
| } |
| _inited = true; |
| } |
| return Status::OK(); |
| } |
| |
| Status IndexFileReader::_init_from(int32_t read_buffer_size, const io::IOContext* io_ctx) { |
| auto index_file_full_path = InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix); |
| |
| try { |
| CLuceneError err; |
| CL_NS(store)::IndexInput* index_input = nullptr; |
| |
| // 1. get file size from meta |
| int64_t file_size = -1; |
| if (_idx_file_info.has_index_size()) { |
| file_size = _idx_file_info.index_size(); |
| } |
| file_size = file_size == 0 ? -1 : file_size; |
| |
| DBUG_EXECUTE_IF("file_size_not_in_rowset_meta ", { |
| if (file_size == -1) { |
| return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>( |
| "CLuceneError occur file size = -1, file is {}", index_file_full_path); |
| } |
| }) |
| |
| DCHECK(_fs != nullptr) << "file system is nullptr, index_file_full_path: " |
| << index_file_full_path; |
| // 2. open file |
| auto ok = |
| DorisFSDirectory::FSIndexInput::open(_fs, index_file_full_path.c_str(), index_input, |
| err, read_buffer_size, file_size, _tablet_id); |
| if (!ok) { |
| if (err.number() == CL_ERR_FileNotFound) { |
| return Status::Error<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>( |
| "inverted index file {} is not found.", index_file_full_path); |
| } else if (err.number() == CL_ERR_EmptyIndexSegment) { |
| return Status::Error<ErrorCode::INVERTED_INDEX_BYPASS>( |
| "inverted index file {} is empty.", index_file_full_path); |
| } |
| return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>( |
| "CLuceneError occur when open idx file {}, error msg: {}", index_file_full_path, |
| err.what()); |
| } |
| _stream = std::unique_ptr<CL_NS(store)::IndexInput>(index_input); |
| _stream->setIoContext(io_ctx); |
| _stream->setIndexFile(true); |
| |
| // 3. read file |
| int32_t version = _stream->readInt(); // Read version number |
| if (version >= InvertedIndexStorageFormatPB::V2) { |
| DCHECK(version == _storage_format); |
| int32_t numIndices = _stream->readInt(); // Read number of indices |
| |
| for (int32_t i = 0; i < numIndices; ++i) { |
| int64_t indexId = _stream->readLong(); // Read index ID |
| int32_t suffix_length = _stream->readInt(); // Read suffix length |
| std::vector<uint8_t> suffix_data(suffix_length); |
| _stream->readBytes(suffix_data.data(), suffix_length); |
| std::string suffix_str(suffix_data.begin(), suffix_data.end()); |
| |
| int32_t numFiles = _stream->readInt(); // Read number of files in the index |
| |
| auto fileEntries = std::make_unique<EntriesType>(); |
| fileEntries->reserve(numFiles); |
| |
| for (int32_t j = 0; j < numFiles; ++j) { |
| int32_t file_name_length = _stream->readInt(); |
| std::string file_name(file_name_length, '\0'); |
| _stream->readBytes(reinterpret_cast<uint8_t*>(file_name.data()), |
| file_name_length); |
| auto entry = std::make_unique<ReaderFileEntry>(); |
| entry->file_name = std::move(file_name); |
| entry->offset = _stream->readLong(); |
| entry->length = _stream->readLong(); |
| fileEntries->emplace(entry->file_name, std::move(entry)); |
| } |
| |
| _indices_entries.emplace(std::make_pair(indexId, std::move(suffix_str)), |
| std::move(fileEntries)); |
| } |
| } else { |
| return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>( |
| "unknown inverted index format {}", version); |
| } |
| } catch (CLuceneError& err) { |
| if (_stream != nullptr) { |
| try { |
| _stream->close(); |
| } catch (CLuceneError& err) { |
| return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>( |
| "CLuceneError occur when close idx file {}, error msg: {}", |
| index_file_full_path, err.what()); |
| } |
| } |
| return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>( |
| "CLuceneError occur when init idx file {}, error msg: {}", index_file_full_path, |
| err.what()); |
| } |
| return Status::OK(); |
| } |
| |
| Status IndexFileReader::_init_snii(const io::IOContext* io_ctx) { |
| auto index_file_full_path = InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix); |
| int64_t file_size = -1; |
| if (_idx_file_info.has_index_size()) { |
| file_size = _idx_file_info.index_size(); |
| } |
| file_size = file_size == 0 ? -1 : file_size; |
| |
| io::FileReaderOptions opts; |
| opts.cache_type = config::enable_file_cache ? io::FileCachePolicy::FILE_BLOCK_CACHE |
| : io::FileCachePolicy::NO_CACHE; |
| opts.is_doris_table = true; |
| opts.file_size = file_size; |
| opts.tablet_id = _tablet_id; |
| io::FileReaderSPtr reader; |
| // A rowset written before any index existed has no container at all. The |
| // filesystem reports that as a plain NOT_FOUND; translate it to the |
| // index-specific code the way the V1/V2 path does, because callers |
| // (IndexBuilder's BUILD INDEX rewrite) distinguish "no container yet, build |
| // everything fresh" from a real IO failure by exactly that code. |
| if (const Status open_status = _fs->open_file(index_file_full_path, &reader, &opts); |
| !open_status.ok()) { |
| if (open_status.is<ErrorCode::NOT_FOUND>()) { |
| return Status::Error<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND, false>( |
| "inverted index file {} is not found.", index_file_full_path); |
| } |
| return open_status; |
| } |
| // With NO_CACHE on a remote filesystem there is no CachedRemoteFileReader to |
| // account physical remote bytes, so the adapter must count its own reads. |
| const bool direct_remote_io = opts.cache_type == io::FileCachePolicy::NO_CACHE && |
| _fs->type() != io::FileSystemType::LOCAL; |
| _snii_file_reader = std::make_shared<snii_doris::DorisSniiFileReader>( |
| std::move(reader), /*io_ctx=*/nullptr, direct_remote_io); |
| _snii_segment_reader = std::make_unique<doris::snii::reader::SniiSegmentReader>(); |
| io::IOContext meta_io_ctx; |
| if (io_ctx != nullptr) { |
| meta_io_ctx = *io_ctx; |
| } |
| meta_io_ctx.is_inverted_index = true; |
| meta_io_ctx.is_index_data = true; |
| snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(&meta_io_ctx); |
| RETURN_IF_ERROR(doris::snii::reader::SniiSegmentReader::open(_snii_file_reader.get(), |
| _snii_segment_reader.get())); |
| return Status::OK(); |
| } |
| |
| Result<InvertedIndexDirectoryMap> IndexFileReader::get_all_directories() { |
| if (_storage_format == InvertedIndexStorageFormatPB::SNII) { |
| return ResultError(Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>( |
| "SNII format does not expose CLucene directories")); |
| } |
| InvertedIndexDirectoryMap res; |
| std::shared_lock<std::shared_mutex> lock(_mutex); // Lock for reading |
| for (auto& [index, _] : _indices_entries) { |
| auto&& [index_id, index_suffix] = index; |
| LOG(INFO) << "index_id:" << index_id << " index_suffix:" << index_suffix; |
| auto ret = _open(index_id, index_suffix); |
| if (!ret.has_value()) { |
| return ResultError(ret.error()); |
| } |
| res.emplace(std::make_pair(index_id, index_suffix), std::move(ret.value())); |
| } |
| return res; |
| } |
| |
| Result<std::unique_ptr<DorisCompoundReader, DirectoryDeleter>> IndexFileReader::_open( |
| int64_t index_id, const std::string& index_suffix, const io::IOContext* io_ctx) const { |
| std::unique_ptr<DorisCompoundReader, DirectoryDeleter> compound_reader; |
| |
| if (_storage_format == InvertedIndexStorageFormatPB::SNII) { |
| // A blob logical index is a named-sub-file table over the container, and |
| // a compound reader is a named-sub-file table over a stream -- the same |
| // shape. The offsets recorded in the directory are ABSOLUTE container |
| // offsets, exactly like a V2 compound entry, so the sub-files need no |
| // rebasing and DorisCompoundReader is reused unchanged. |
| const auto index_file_path = |
| InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix); |
| EntriesType entries; |
| int64_t container_size = 0; |
| // The lock spans every use of _snii_segment_reader state, not just the |
| // lookup: `entry` points into the reader's decoded directory, and every |
| // other SNII accessor on this class holds the lock across the whole use. |
| // Narrowing it here would make this the one site whose safety rests on |
| // "the segment reader is never reset after init" rather than on the lock. |
| { |
| std::shared_lock<std::shared_mutex> lock(_mutex); |
| if (_snii_segment_reader == nullptr) { |
| return ResultError(Status::Error<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>( |
| "SNII index file {} is not opened", index_file_path)); |
| } |
| const doris::snii::format::LogicalIndexMetadataRef* entry = nullptr; |
| RETURN_IF_ERROR_RESULT(_snii_segment_reader->blob_entry(cast_set<uint64_t>(index_id), |
| index_suffix, &entry)); |
| DORIS_CHECK(entry != nullptr); |
| // Only an ANN index is served through a CLucene directory. A BKD blob |
| // has its own reader and must not be reachable this way, or a caller |
| // would get a directory over bytes no CLucene code can parse. |
| if (entry->kind != doris::snii::format::LogicalIndexKind::kAnn) { |
| return ResultError(Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>( |
| "SNII logical index {} is not an ANN blob; it has no CLucene directory", |
| index_id)); |
| } |
| // Blob extents were bounded against the container at open time |
| // (SniiSegmentReader::validate_blob_files), so they are safe to hand |
| // to the compound reader as-is. |
| for (const auto& blob : entry->files) { |
| auto file_entry = std::make_unique<ReaderFileEntry>(); |
| file_entry->file_name = blob.name; |
| file_entry->offset = cast_set<int64_t>(blob.offset); |
| file_entry->length = cast_set<int64_t>(blob.length); |
| entries.emplace(blob.name, std::move(file_entry)); |
| } |
| container_size = get_inverted_file_size(); |
| } |
| |
| CLuceneError err; |
| CL_NS(store)::IndexInput* index_input = nullptr; |
| // The container size is already resident -- init() opened the file to read |
| // its directory. Passing -1 here would make the filesystem re-discover it, |
| // which is a stat() locally and a HeadObject round trip on S3, on every |
| // cold ANN index load. |
| if (!DorisFSDirectory::FSIndexInput::open(_fs, index_file_path.c_str(), index_input, err, |
| _read_buffer_size, container_size, _tablet_id)) { |
| return ResultError(Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>( |
| "CLuceneError occur when open SNII container {}, error msg: {}", |
| index_file_path, err.what())); |
| } |
| compound_reader.reset( |
| new DorisCompoundReader(index_input, entries, _read_buffer_size, io_ctx)); |
| return compound_reader; |
| } |
| |
| if (_storage_format == InvertedIndexStorageFormatPB::V1) { |
| auto index_file_path = InvertedIndexDescriptor::get_index_file_path_v1( |
| _index_path_prefix, index_id, index_suffix); |
| try { |
| CLuceneError err; |
| CL_NS(store)::IndexInput* index_input = nullptr; |
| |
| // 1. get file size from meta |
| int64_t file_size = -1; |
| if (_idx_file_info.index_info_size() > 0) { |
| for (const auto& idx_info : _idx_file_info.index_info()) { |
| if (index_id == idx_info.index_id() && |
| index_suffix == idx_info.index_suffix()) { |
| file_size = idx_info.index_file_size(); |
| break; |
| } |
| } |
| } |
| file_size = file_size == 0 ? -1 : file_size; |
| DBUG_EXECUTE_IF("file_size_not_in_rowset_meta ", { |
| if (file_size == -1) { |
| return ResultError(Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>( |
| "CLuceneError occur file size = -1, file is {}", index_file_path)); |
| } |
| }) |
| DCHECK(_fs != nullptr) |
| << "file system is nullptr, index_file_path: " << index_file_path; |
| // 2. open file |
| auto ok = DorisFSDirectory::FSIndexInput::open(_fs, index_file_path.c_str(), |
| index_input, err, _read_buffer_size, |
| file_size, _tablet_id); |
| if (!ok) { |
| // now index_input = nullptr |
| if (err.number() == CL_ERR_FileNotFound) { |
| return ResultError(Status::Error<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>( |
| "inverted index file {} is not found.", index_file_path)); |
| } |
| return ResultError(Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>( |
| "CLuceneError occur when open idx file {}, error msg: {}", index_file_path, |
| err.what())); |
| } |
| |
| // 3. read file in DorisCompoundReader |
| compound_reader.reset(new DorisCompoundReader(index_input, _read_buffer_size)); |
| } catch (CLuceneError& err) { |
| return ResultError(Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>( |
| "CLuceneError occur when open idx file {}, error msg: {}", index_file_path, |
| err.what())); |
| } |
| } else { |
| std::shared_lock<std::shared_mutex> lock(_mutex); // Lock for reading |
| if (_stream == nullptr) { |
| return ResultError(Status::Error<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>( |
| "CLuceneError occur when open idx file {}, stream is nullptr", |
| InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix))); |
| } |
| |
| // Check if the specified index exists |
| auto index_it = _indices_entries.find(std::make_pair(index_id, index_suffix)); |
| if (index_it == _indices_entries.end()) { |
| std::ostringstream errMsg; |
| errMsg << "No index with id " << index_id << " found"; |
| return ResultError(Status::Error<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>( |
| "CLuceneError occur when open idx file {}, error msg: {}", |
| InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix), |
| errMsg.str())); |
| } |
| // Need to clone resource here, because index searcher cache need it. |
| compound_reader.reset(new DorisCompoundReader(_stream->clone(), *index_it->second, |
| _read_buffer_size, io_ctx)); |
| } |
| return compound_reader; |
| } |
| |
| Result<std::unique_ptr<doris::snii::reader::LogicalIndexReader>> IndexFileReader::open_snii_index( |
| const TabletIndex* index_meta, const io::IOContext* io_ctx, |
| doris::snii::reader::LogicalIndexOpenMode open_mode) const { |
| DCHECK(_storage_format == InvertedIndexStorageFormatPB::SNII); |
| std::shared_lock<std::shared_mutex> lock(_mutex); |
| if (_snii_segment_reader == nullptr) { |
| return ResultError(Status::Error<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>( |
| "SNII index file {} is not opened", |
| InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix))); |
| } |
| io::IOContext meta_io_ctx; |
| if (io_ctx != nullptr) { |
| meta_io_ctx = *io_ctx; |
| } |
| meta_io_ctx.is_inverted_index = true; |
| meta_io_ctx.is_index_data = true; |
| snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(&meta_io_ctx); |
| |
| auto logical_reader = std::make_unique<doris::snii::reader::LogicalIndexReader>(); |
| auto status = _snii_segment_reader->open_index(cast_set<uint64_t>(index_meta->index_id()), |
| index_meta->get_index_suffix(), |
| logical_reader.get(), open_mode); |
| auto doris_status = status; |
| if (!doris_status.ok()) { |
| return ResultError(doris_status); |
| } |
| return logical_reader; |
| } |
| |
| Result<std::unique_ptr<doris::snii::bkd::BkdSearcher>> IndexFileReader::open_snii_bkd_index( |
| const TabletIndex* index_meta, const io::IOContext* io_ctx) const { |
| DCHECK(_storage_format == InvertedIndexStorageFormatPB::SNII); |
| std::shared_lock<std::shared_mutex> lock(_mutex); |
| if (_snii_segment_reader == nullptr) { |
| return ResultError(Status::Error<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>( |
| "SNII index file {} is not opened", |
| InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix))); |
| } |
| io::IOContext meta_io_ctx; |
| if (io_ctx != nullptr) { |
| meta_io_ctx = *io_ctx; |
| } |
| meta_io_ctx.is_inverted_index = true; |
| meta_io_ctx.is_index_data = true; |
| snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(&meta_io_ctx); |
| |
| const doris::snii::format::LogicalIndexMetadataRef* entry = nullptr; |
| RETURN_IF_ERROR_RESULT(_snii_segment_reader->blob_entry( |
| cast_set<uint64_t>(index_meta->index_id()), index_meta->get_index_suffix(), &entry)); |
| |
| // Placement is the container's decision, so every extent comes from the |
| // sealed directory rather than from anything the producer remembered. |
| doris::snii::bkd::BkdSections sections; |
| auto searcher = std::make_unique<doris::snii::bkd::BkdSearcher>(); |
| for (const doris::snii::format::NamedBlobFileRef& blob : entry->files) { |
| if (blob.name == "bkd_data") { |
| sections.data_offset = blob.offset; |
| sections.data_length = blob.length; |
| } else if (blob.name == "bkd_index") { |
| sections.index_offset = blob.offset; |
| sections.index_length = blob.length; |
| } else if (blob.name == "bkd_nulls") { |
| searcher->null_bitmap_offset = blob.offset; |
| searcher->null_bitmap_length = blob.length; |
| } |
| } |
| RETURN_IF_ERROR_RESULT(doris::snii::bkd::BkdReader::open(_snii_segment_reader->reader(), |
| sections, &searcher->reader)); |
| return searcher; |
| } |
| |
| Status IndexFileReader::prepare_snii_rewrite_snapshot( |
| const std::vector<doris::snii::reader::LogicalIndexKey>& keep, uint64_t segment_doc_count, |
| doris::snii::reader::SniiRewriteSnapshot* out) const { |
| DCHECK(_storage_format == InvertedIndexStorageFormatPB::SNII); |
| std::shared_lock<std::shared_mutex> lock(_mutex); |
| if (_snii_segment_reader == nullptr) { |
| return Status::Error<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>( |
| "SNII index file {} is not opened", |
| InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix)); |
| } |
| return _snii_segment_reader->prepare_rewrite_snapshot(keep, segment_doc_count, out); |
| } |
| |
| Result<std::unique_ptr<DorisCompoundReader, DirectoryDeleter>> IndexFileReader::open( |
| const TabletIndex* index_meta, const io::IOContext* io_ctx) const { |
| auto index_id = index_meta->index_id(); |
| auto index_suffix = index_meta->get_index_suffix(); |
| return _open(index_id, index_suffix, io_ctx); |
| } |
| |
| std::string IndexFileReader::get_index_file_cache_key(const TabletIndex* index_meta) const { |
| return InvertedIndexDescriptor::get_index_file_cache_key( |
| _index_path_prefix, index_meta->index_id(), index_meta->get_index_suffix()); |
| } |
| |
| std::string IndexFileReader::get_index_file_path(const TabletIndex* index_meta) const { |
| if (_storage_format == InvertedIndexStorageFormatPB::V1) { |
| return InvertedIndexDescriptor::get_index_file_path_v1( |
| _index_path_prefix, index_meta->index_id(), index_meta->get_index_suffix()); |
| } |
| return InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix); |
| } |
| |
| Status IndexFileReader::index_file_exist(const TabletIndex* index_meta, bool* res) const { |
| if (_storage_format == InvertedIndexStorageFormatPB::V1) { |
| auto index_file_path = InvertedIndexDescriptor::get_index_file_path_v1( |
| _index_path_prefix, index_meta->index_id(), index_meta->get_index_suffix()); |
| return _fs->exists(index_file_path, res); |
| } else if (_storage_format == InvertedIndexStorageFormatPB::SNII) { |
| auto index_file_path = InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix); |
| RETURN_IF_ERROR(_fs->exists(index_file_path, res)); |
| if (!*res) { |
| return Status::OK(); |
| } |
| std::shared_lock<std::shared_mutex> lock(_mutex); |
| if (_snii_segment_reader == nullptr) { |
| // The container is on disk but this reader never opened it, so we |
| // cannot tell whether the index is in there. Answering "absent" |
| // would make a BUILD INDEX rewrite drop it silently; report the real |
| // condition instead, as the V2 branch below does. |
| *res = false; |
| return Status::Error<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>( |
| "SNII idx file {} exists but is not opened", |
| InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix)); |
| } |
| return _snii_segment_reader->index_exists(cast_set<uint64_t>(index_meta->index_id()), |
| index_meta->get_index_suffix(), res); |
| } else { |
| std::shared_lock<std::shared_mutex> lock(_mutex); // Lock for reading |
| if (_stream == nullptr) { |
| *res = false; |
| return Status::Error<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>( |
| "idx file {} is not opened", |
| InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix)); |
| } |
| // Check if the specified index exists |
| auto index_it = _indices_entries.find( |
| std::make_pair(index_meta->index_id(), index_meta->get_index_suffix())); |
| if (index_it == _indices_entries.end()) { |
| *res = false; |
| } else { |
| *res = true; |
| } |
| } |
| return Status::OK(); |
| } |
| |
| Status IndexFileReader::has_null(const TabletIndex* index_meta, bool* res) const { |
| if (_storage_format == InvertedIndexStorageFormatPB::V1) { |
| *res = true; |
| return Status::OK(); |
| } |
| if (_storage_format == InvertedIndexStorageFormatPB::SNII) { |
| std::shared_lock<std::shared_mutex> lock(_mutex); |
| if (_snii_segment_reader == nullptr) { |
| return Status::Error<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>( |
| "SNII index file {} is not opened", |
| InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix)); |
| } |
| io::IOContext meta_io_ctx; |
| meta_io_ctx.is_inverted_index = true; |
| meta_io_ctx.is_index_data = true; |
| snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(&meta_io_ctx); |
| |
| doris::snii::format::SectionRefs section_refs; |
| RETURN_IF_ERROR(_snii_segment_reader->section_refs_for_index( |
| cast_set<uint64_t>(index_meta->index_id()), index_meta->get_index_suffix(), |
| §ion_refs)); |
| *res = section_refs.null_bitmap.length > 0; |
| return Status::OK(); |
| } |
| std::shared_lock<std::shared_mutex> lock(_mutex); // Lock for reading |
| if (_stream == nullptr) { |
| return Status::Error<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>( |
| "idx file {} is not opened", |
| InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix)); |
| } |
| // Check if the specified index exists |
| auto index_it = _indices_entries.find( |
| std::make_pair(index_meta->index_id(), index_meta->get_index_suffix())); |
| if (index_it == _indices_entries.end()) { |
| *res = false; |
| } else { |
| const auto& entries = index_it->second; |
| auto entry_it = |
| entries->find(InvertedIndexDescriptor::get_temporary_null_bitmap_file_name()); |
| if (entry_it == entries->end()) { |
| *res = false; |
| return Status::OK(); |
| } |
| const auto& e = entry_it->second; |
| // roaring bitmap cookie header size is 5 |
| if (e->length <= 5) { |
| *res = false; |
| } else { |
| *res = true; |
| } |
| } |
| return Status::OK(); |
| } |
| |
| void IndexFileReader::debug_file_entries() { |
| std::shared_lock<std::shared_mutex> lock(_mutex); // Lock for reading |
| for (const auto& index : _indices_entries) { |
| LOG(INFO) << "index_id:" << index.first.first; |
| const auto& index_entries = index.second; |
| for (const auto& entry : *index_entries) { |
| const auto& file_entry = entry.second; |
| LOG(INFO) << "file entry name:" << file_entry->file_name |
| << " length:" << file_entry->length << " offset:" << file_entry->offset; |
| } |
| } |
| } |
| |
| } // namespace doris::segment_v2 |
