// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

#include <CLucene.h>
#include <CLucene/config/repl_wchar.h>
#include <gen_cpp/PaloInternalService_types.h>
#include <gen_cpp/olap_file.pb.h>
#include <gflags/gflags.h>

#include <filesystem>
#include <fstream>
#include <iostream>
#include <memory>
#include <nlohmann/json.hpp>
#include <roaring/roaring.hh>
#include <sstream>
#include <string>
#include <vector>

#include "io/fs/file_reader.h"
#ifdef __clang__
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wshadow-field"
#endif
// clang-format off
#include "common/compile_check_avoid_begin.h"
#include "CLucene/analysis/standard95/StandardAnalyzer.h"
#include "common/compile_check_avoid_end.h"
// clang-format on
#ifdef __clang__
#pragma clang diagnostic pop
#endif
#include "io/fs/local_file_system.h"
#include "runtime/thread_context.h"
#include "storage/index/index_file_reader.h"
#include "storage/index/index_file_writer.h"
#include "storage/index/inverted/inverted_index_compound_reader.h"
#include "storage/index/inverted/inverted_index_desc.h"
#include "storage/index/inverted/inverted_index_fs_directory.h"
#include "storage/index/inverted/query/conjunction_query.h"
#include "storage/tablet/tablet_schema.h"

using doris::segment_v2::DorisCompoundReader;
using doris::segment_v2::DorisFSDirectoryFactory;
using doris::segment_v2::IndexFileWriter;
using doris::segment_v2::InvertedIndexDescriptor;
using doris::segment_v2::IndexFileReader;
using doris::io::FileInfo;
using doris::TabletIndex;
using namespace doris::segment_v2;
using namespace lucene::analysis;
using namespace lucene::index;
using namespace lucene::util;
using namespace lucene::search;

DEFINE_string(operation, "",
              "valid operation: show_nested_files,check_terms,term_query,debug_index_compaction");

DEFINE_string(directory, "./", "inverted index file directory");
DEFINE_string(idx_file_name, "", "inverted index file name");
DEFINE_string(idx_file_path, "", "inverted index file path");
DEFINE_string(data_file_path, "", "inverted index data path");
DEFINE_string(term, "", "inverted index term to query");
DEFINE_string(column_name, "", "inverted index column_name to query");
DEFINE_string(pred_type, "", "inverted index term query predicate, eq/lt/gt/le/ge/match etc.");
DEFINE_bool(print_row_id, false, "print row id when query terms");
DEFINE_bool(print_doc_id, false, "print doc id when check terms stats");
// only for debug index compaction
DEFINE_int64(idx_id, -1, "inverted index id");
DEFINE_string(src_idx_dirs_file, "", "source segment index files");
DEFINE_string(dest_idx_dirs_file, "", "destination segment index files");
DEFINE_string(dest_seg_num_rows_file, "", "destination segment number of rows");
DEFINE_string(tablet_path, "", "tablet path");
DEFINE_string(trans_vec_file, "", "rowid conversion map file");

std::string get_usage(const std::string& progname) {
    std::stringstream ss;
    ss << progname << " is the Doris inverted index file tool.\n";
    ss << "Usage:\n";
    ss << "./index_tool --operation=show_nested_files --idx_file_path=path/to/file\n";
    ss << "./index_tool --operation=check_terms_stats --idx_file_path=path/to/file "
          "--print_doc_id\n";
    ss << "./index_tool --operation=term_query --directory=directory "
          "--idx_file_name=file --print_row_id --term=term --column_name=column_name "
          "--pred_type=eq/lt/gt/le/ge/match etc\n";
    ss << "*** debug_index_compaction operation is only for offline debug index compaction, do not "
          "use in production ***\n";
    ss << "./index_tool --operation=debug_index_compaction --idx_id=index_id "
          "--src_idx_dirs_file=path/to/file --dest_idx_dirs_file=path/to/file "
          "--dest_seg_num_rows_file=path/to/file --tablet_path=path/to/tablet "
          "--trans_vec_file=path/to/file\n";
    ss << "./index_tool --operation=write_index_v2 --idx_file_path=path/to/index "
          "--data_file_path=data/to/index\n";
    ss << "./index_tool --operation=show_nested_files_v2 --idx_file_path=path/to/file\n";
    ss << "./index_tool --operation=check_terms_stats_v2 --idx_file_path=path/to/file "
          "--idx_id=index_id\n";
    return ss.str();
}

std::vector<std::string> split(const std::string& s, char delimiter) {
    std::vector<std::string> tokens;
    std::string token;
    std::istringstream tokenStream(s);
    while (getline(tokenStream, token, delimiter)) {
        tokens.push_back(token);
    }
    return tokens;
}

void search(lucene::store::Directory* dir, std::string& field, std::string& token,
            std::string& pred) {
    lucene::index::IndexReader* reader = lucene::index::IndexReader::open(dir);

    lucene::index::IndexReader* newreader = reader->reopen();
    if (newreader != reader) {
        reader->close();
        _CLDELETE(reader);
        reader = newreader;
    }
    auto s = std::make_shared<IndexSearcher>(reader);
    std::unique_ptr<lucene::search::Query> query;

    std::cout << "version: " << (int32_t)(reader->getIndexVersion()) << std::endl;

    std::wstring field_ws(field.begin(), field.end());
    if (pred == "match_all") {
    } else if (pred == "match_phrase") {
        std::vector<std::string> terms = split(token, '|');
        auto* phrase_query = new lucene::search::PhraseQuery();
        for (auto& term : terms) {
            std::wstring term_ws = StringUtil::string_to_wstring(term);
            auto* t = _CLNEW lucene::index::Term(field_ws.c_str(), term_ws.c_str());
            phrase_query->add(t);
            _CLDECDELETE(t);
        }
        query.reset(phrase_query);
    } else {
        std::wstring token_ws(token.begin(), token.end());
        lucene::index::Term* term = _CLNEW lucene::index::Term(field_ws.c_str(), token_ws.c_str());
        if (pred == "eq" || pred == "match") {
            query.reset(new lucene::search::TermQuery(term));
        } else if (pred == "lt") {
            query.reset(new lucene::search::RangeQuery(nullptr, term, false));
        } else if (pred == "gt") {
            query.reset(new lucene::search::RangeQuery(term, nullptr, false));
        } else if (pred == "le") {
            query.reset(new lucene::search::RangeQuery(nullptr, term, true));
        } else if (pred == "ge") {
            query.reset(new lucene::search::RangeQuery(term, nullptr, true));
        } else {
            std::cout << "invalid predicate type:" << pred << std::endl;
            exit(-1);
        }
        _CLDECDELETE(term);
    }

    int32_t total = 0;
    if (pred == "match_all") {
        roaring::Roaring result;
        std::vector<std::string> terms = split(token, '|');

        doris::OlapReaderStatistics stats;
        doris::RuntimeState runtime_state;
        doris::TQueryOptions query_options;
        query_options.inverted_index_max_expansions = 50;
        runtime_state.set_query_options(query_options);
        doris::io::IOContext io_ctx;

        IndexQueryContextPtr context = std::make_shared<IndexQueryContext>();
        context->runtime_state = &runtime_state;
        context->stats = &stats;
        context->io_ctx = &io_ctx;

        ConjunctionQuery conjunct_query(s, context);
        InvertedIndexQueryInfo query_info;
        query_info.field_name = field_ws;
        for (auto& term : terms) {
            doris::segment_v2::TermInfo term_info;
            term_info.term = term;
            query_info.term_infos.push_back(term_info);
        }
        conjunct_query.add(query_info);
        conjunct_query.search(result);

        total += result.cardinality();
    } else {
        roaring::Roaring result;
        s->_search(query.get(), [&result](const int32_t docid, const float_t /*score*/) {
            // docid equal to rowid in segment
            result.add(docid);
            if (FLAGS_print_row_id) {
                printf("RowID is %d\n", docid);
            }
        });
        total += result.cardinality();
    }
    std::cout << "Term queried count:" << total << std::endl;

    s->close();
    reader->close();
    _CLLDELETE(reader);
}

void check_terms_stats(lucene::store::Directory* dir) {
    lucene::index::IndexReader* r = lucene::index::IndexReader::open(dir);

    printf("Max Docs: %d\n", r->maxDoc());
    printf("Num Docs: %d\n", r->numDocs());

    int64_t ver = r->getCurrentVersion(dir);
    printf("Current Version: %f\n", (float_t)ver);

    TermEnum* te = r->terms();
    int32_t nterms;
    for (nterms = 0; te->next(); nterms++) {
        /* empty */
        std::string token =
                lucene_wcstoutf8string(te->term(false)->text(), te->term(false)->textLength());
        std::string field = lucene_wcstoutf8string(te->term(false)->field(),
                                                   lenOfString(te->term(false)->field()));

        printf("Field: %s ", field.c_str());
        printf("Term: %s ", token.c_str());
        printf("Freq: %d\n", te->docFreq());
        if (FLAGS_print_doc_id) {
            TermDocs* td = r->termDocs(te->term());
            while (td->next()) {
                printf("DocID: %d ", td->doc());
                printf("TermFreq: %d\n", td->freq());
            }
            _CLLDELETE(td);
        }
    }
    printf("Term count: %d\n\n", nterms);
    te->close();
    _CLLDELETE(te);

    r->close();
    _CLLDELETE(r);
}

std::unique_ptr<DorisCompoundReader> get_compound_reader(std::string file_path) {
    CLuceneError err;
    CL_NS(store)::IndexInput* index_input = nullptr;
    auto ok = DorisFSDirectory::FSIndexInput::open(doris::io::global_local_filesystem(),
                                                   file_path.c_str(), index_input, err, 4096, -1);
    if (!ok) {
        // now index_input = nullptr
        if (err.number() == CL_ERR_FileNotFound) {
            std::cerr << "file " << file_path << " not found" << std::endl;
            exit(-1);
        }
        std::cerr << "file " << file_path << " open error:" << err.what() << std::endl;
        exit(-1);
    }

    return std::make_unique<DorisCompoundReader>(index_input, 4096);
}

int main(int argc, char** argv) {
    SCOPED_INIT_THREAD_CONTEXT();

    std::string usage = get_usage(argv[0]);
    gflags::SetUsageMessage(usage);
    google::ParseCommandLineFlags(&argc, &argv, true);

    if (FLAGS_operation == "show_nested_files") {
        if (FLAGS_idx_file_path == "") {
            std::cout << "no file flag for show " << std::endl;
            return -1;
        }
        std::filesystem::path p(FLAGS_idx_file_path);
        std::string dir_str = p.parent_path().string();
        std::string file_str = p.filename().string();
        auto fs = doris::io::global_local_filesystem();
        bool is_exists = false;
        const auto file_path = dir_str + "/" + file_str;
        if (!(fs->exists(file_path, &is_exists).ok()) || !is_exists) {
            std::cerr << "file " << file_path << " not found" << std::endl;
            return -1;
        }
        std::unique_ptr<DorisCompoundReader> reader;
        try {
            reader = get_compound_reader(file_path);
            std::vector<std::string> files;
            std::cout << "Nested files for " << file_str << std::endl;
            std::cout << "==================================" << std::endl;
            reader->list(&files);
            for (auto& file : files) {
                std::cout << file << std::endl;
            }
            reader->close();
        } catch (CLuceneError& err) {
            std::cerr << "error occurred when show files: " << err.what() << std::endl;
            if (reader) {
                reader->close();
            }
            return -1;
        }
    } else if (FLAGS_operation == "check_terms_stats") {
        if (FLAGS_idx_file_path == "") {
            std::cout << "no file flag for check " << std::endl;
            return -1;
        }
        std::filesystem::path p(FLAGS_idx_file_path);
        std::string dir_str = p.parent_path().string();
        std::string file_str = p.filename().string();
        auto fs = doris::io::global_local_filesystem();
        bool is_exists = false;
        const auto file_path = dir_str + "/" + file_str;
        if (!(fs->exists(file_path, &is_exists).ok()) || !is_exists) {
            std::cerr << "file " << file_path << " not found" << std::endl;
            return -1;
        }
        std::unique_ptr<DorisCompoundReader> reader;
        try {
            reader = get_compound_reader(file_path);
            std::cout << "Term statistics for " << file_str << std::endl;
            std::cout << "==================================" << std::endl;
            check_terms_stats(reader.get());
            reader->close();
        } catch (CLuceneError& err) {
            std::cerr << "error occurred when check_terms_stats: " << err.what() << std::endl;
            if (reader) {
                reader->close();
            }
            return -1;
        }
    } else if (FLAGS_operation == "term_query") {
        if (FLAGS_directory == "" || FLAGS_term == "" || FLAGS_column_name == "" ||
            FLAGS_pred_type == "") {
            std::cerr << "invalid params for term_query " << std::endl;
            return -1;
        }
        auto fs = doris::io::global_local_filesystem();
        std::unique_ptr<DorisCompoundReader> reader;
        try {
            if (FLAGS_idx_file_name == "") {
                //try to search from directory's all files
                std::vector<doris::io::FileInfo> files;
                bool exists = false;
                std::filesystem::path root_dir(FLAGS_directory);
                doris::Status status = fs->list(root_dir, true, &files, &exists);
                if (!status.ok()) {
                    std::cerr << "can't search from directory's all files,err : " << status
                              << std::endl;
                    return -1;
                }
                if (!exists) {
                    std::cerr << FLAGS_directory << " is not exists" << std::endl;
                    return -1;
                }
                for (auto& f : files) {
                    try {
                        auto file_str = f.file_name;
                        if (!file_str.ends_with(".idx")) {
                            continue;
                        }
                        const auto file_path = FLAGS_directory + "/" + file_str;
                        reader = get_compound_reader(file_path);
                        std::cout << "Search " << FLAGS_column_name << ":" << FLAGS_term << " from "
                                  << file_str << std::endl;
                        std::cout << "==================================" << std::endl;
                        search(reader.get(), FLAGS_column_name, FLAGS_term, FLAGS_pred_type);
                        reader->close();
                    } catch (CLuceneError& err) {
                        std::cerr << "error occurred when search file: " << f.file_name
                                  << ", error:" << err.what() << std::endl;
                        if (reader) {
                            reader->close();
                        }
                        return -1;
                    }
                }
            } else {
                bool is_exists = false;
                auto file_path = FLAGS_directory + "/" + FLAGS_idx_file_name;
                if (!(fs->exists(file_path, &is_exists).ok()) || !is_exists) {
                    std::cerr << "file " << file_path << " not found" << std::endl;
                    return -1;
                }
                reader = get_compound_reader(file_path);
                std::cout << "Search " << FLAGS_column_name << ":" << FLAGS_term << " from "
                          << FLAGS_idx_file_name << std::endl;
                std::cout << "==================================" << std::endl;
                try {
                    search(reader.get(), FLAGS_column_name, FLAGS_term, FLAGS_pred_type);
                    reader->close();
                } catch (CLuceneError& err) {
                    std::cerr << "error occurred when search file: " << FLAGS_idx_file_name
                              << ", error:" << err.what() << std::endl;
                    if (reader) {
                        reader->close();
                    }
                    return -1;
                }
            }
        } catch (CLuceneError& err) {
            std::cerr << "error occurred when check_terms_stats: " << err.what() << std::endl;
            if (reader) {
                reader->close();
            }
            return -1;
        }
    } else if (FLAGS_operation == "debug_index_compaction") {
        // only for debug index compaction, do not use in production
        if (FLAGS_idx_id <= 0 || FLAGS_src_idx_dirs_file == "" || FLAGS_dest_idx_dirs_file == "" ||
            FLAGS_dest_seg_num_rows_file == "" || FLAGS_tablet_path == "" ||
            FLAGS_trans_vec_file == "") {
            std::cout << "invalid params for debug_index_compaction " << std::endl;
            return -1;
        }

        auto fs = doris::io::global_local_filesystem();

        auto read_file_to_json = [&](const std::string& file, std::string& output) {
            doris::io::FileReaderSPtr file_reader;
            doris::Status status = fs->open_file(file, &file_reader);
            if (!status.ok()) {
                std::cout << "read file " << file << " failed" << std::endl;
                return false;
            }
            size_t fsize = file_reader->size();
            if (fsize > 0) {
                output.resize(fsize);
                size_t bytes_read = 0;
                status = file_reader->read_at(0, {output.data(), fsize}, &bytes_read);
            }
            if (!status.ok()) {
                std::cout << "read file " << file << " failed" << std::endl;
                return false;
            }
            return true;
        };

        int64_t index_id = FLAGS_idx_id;
        std::string tablet_path = FLAGS_tablet_path;
        std::string src_index_dirs_string;
        std::string dest_index_dirs_string;
        std::string dest_segment_num_rows_string;
        std::string trans_vec_string;

        if (!read_file_to_json(FLAGS_src_idx_dirs_file, src_index_dirs_string) ||
            !read_file_to_json(FLAGS_dest_idx_dirs_file, dest_index_dirs_string) ||
            !read_file_to_json(FLAGS_dest_seg_num_rows_file, dest_segment_num_rows_string) ||
            !read_file_to_json(FLAGS_trans_vec_file, trans_vec_string)) {
            return -1;
        }
        std::vector<std::string> src_index_files = nlohmann::json::parse(src_index_dirs_string);
        std::vector<std::string> dest_index_files = nlohmann::json::parse(dest_index_dirs_string);
        std::vector<uint32_t> dest_segment_num_rows =
                nlohmann::json::parse(dest_segment_num_rows_string);
        std::vector<std::vector<std::pair<uint32_t, uint32_t>>> trans_vec =
                nlohmann::json::parse(trans_vec_string);
        int src_segment_num = src_index_files.size();
        int dest_segment_num = dest_index_files.size();

        std::string index_writer_path = tablet_path + "/tmp_index_writer";
        lucene::store::Directory* dir =
                DorisFSDirectoryFactory::getDirectory(fs, index_writer_path.c_str(), false);
        lucene::analysis::SimpleAnalyzer<char> analyzer;
        auto index_writer = _CLNEW lucene::index::IndexWriter(dir, &analyzer, true /* create */,
                                                              true /* closeDirOnShutdown */);
        std::ostream* infoStream = &std::cout;
        index_writer->setInfoStream(infoStream);
        // get compound directory src_index_dirs
        std::vector<lucene::store::Directory*> src_index_dirs(src_segment_num);
        for (int i = 0; i < src_segment_num; ++i) {
            // format: rowsetId_segmentId_indexId.idx
            std::string src_idx_full_name =
                    src_index_files[i] + "_" + std::to_string(index_id) + ".idx";
            const auto file_path = tablet_path + "/" + src_idx_full_name;
            auto reader_ptr = get_compound_reader(file_path);
            DorisCompoundReader* reader = reader_ptr.release();
            src_index_dirs[i] = reader;
        }

        // get dest idx file paths
        std::vector<lucene::store::Directory*> dest_index_dirs(dest_segment_num);
        for (int i = 0; i < dest_segment_num; ++i) {
            // format: rowsetId_segmentId_columnId
            auto path = tablet_path + "/" + dest_index_files[i] + "_" + std::to_string(index_id);
            dest_index_dirs[i] = DorisFSDirectoryFactory::getDirectory(fs, path.c_str(), true);
        }

        index_writer->indexCompaction(src_index_dirs, dest_index_dirs, trans_vec,
                                      dest_segment_num_rows);

        index_writer->close();
        _CLDELETE(index_writer);
        // NOTE: need to ref_cnt-- for dir,
        // when index_writer is destroyed, if closeDir is set, dir will be close
        // _CLDECDELETE(dir) will try to ref_cnt--, when it decreases to 1, dir will be destroyed.
        _CLDECDELETE(dir)
        for (auto d : src_index_dirs) {
            if (d != nullptr) {
                d->close();
                _CLDELETE(d);
            }
        }
        for (auto d : dest_index_dirs) {
            if (d != nullptr) {
                // NOTE: DO NOT close dest dir here, because it will be closed when dest index writer finalize.
                //d->close();
                _CLDELETE(d);
            }
        }

        // delete temporary index_writer_path
        if (!fs->delete_directory(index_writer_path.c_str()).ok()) {
            std::cout << "delete temporary index writer path: " << index_writer_path << " failed."
                      << std::endl;
            return -1;
        }
    } else if (FLAGS_operation == "write_index_v2") {
        if (FLAGS_idx_file_path == "") {
            std::cout << "no index path flag for check " << std::endl;
            return -1;
        }
        if (FLAGS_data_file_path == "") {
            std::cout << "no data file flag for check " << std::endl;
            return -1;
        }

        const std::string rowset_id = "test_rowset";
        constexpr int seg_id = 0;
        std::string name = "test";
        const std::string& file_dir = FLAGS_idx_file_path;
        constexpr int64_t index_id = 1;
        doris::TabletIndexPB index_pb;
        index_pb.set_index_id(index_id);
        TabletIndex index_meta;
        index_meta.init_from_pb(index_pb);

        std::vector<std::string> datas;
        {
            if (std::filesystem::exists(FLAGS_data_file_path) &&
                std::filesystem::file_size(FLAGS_data_file_path) == 0) {
                std::cerr << "Error: File '" << FLAGS_data_file_path << "' is empty." << std::endl;
                return -1;
            } else {
                std::ifstream ifs;
                std::cout << "prepare to load " << FLAGS_data_file_path << std::endl;

                ifs.open(FLAGS_data_file_path);
                if (!ifs) {
                    std::cerr << "Error: Unable to open file '" << FLAGS_data_file_path << "'."
                              << std::endl;
                    return -1;
                } else {
                    std::string line;
                    while (std::getline(ifs, line)) {
                        datas.emplace_back(line);
                    }
                    ifs.close();
                }
            }
        }

        auto fs = doris::io::global_local_filesystem();
        auto index_file_writer = std::make_unique<IndexFileWriter>(
                fs,
                std::string {InvertedIndexDescriptor::get_index_file_path_prefix(
                        doris::local_segment_path(file_dir, rowset_id, seg_id))},
                rowset_id, seg_id, doris::InvertedIndexStorageFormatPB::V2);
        auto st = index_file_writer->open(&index_meta);
        if (!st.has_value()) {
            std::cerr << "IndexFileWriter init error:" << st.error() << std::endl;
            return -1;
        }
        using T = std::decay_t<decltype(st)>;
        auto dir = std::forward<T>(st).value();
        auto analyzer = _CLNEW lucene::analysis::standard95::StandardAnalyzer();
        // auto analyzer = _CLNEW lucene::analysis::SimpleAnalyzer<char>();
        auto indexwriter = _CLNEW lucene::index::IndexWriter(dir.get(), analyzer, true, true);
        indexwriter->setRAMBufferSizeMB(512);
        indexwriter->setMaxFieldLength(0x7FFFFFFFL);
        indexwriter->setMergeFactor(100000000);
        indexwriter->setUseCompoundFile(false);

        auto char_string_reader = _CLNEW lucene::util::SStringReader<char>;

        auto doc = _CLNEW lucene::document::Document();
        auto field_config = (int32_t)(lucene::document::Field::STORE_NO);
        field_config |= (int32_t)(lucene::document::Field::INDEX_NONORMS);
        field_config |= lucene::document::Field::INDEX_TOKENIZED;
        auto field_name = StringUtil::string_to_wstring(name);
        auto field = _CLNEW lucene::document::Field(field_name.c_str(), field_config);
        field->setOmitTermFreqAndPositions(false);
        doc->add(*field);

        for (int32_t j = 0; j < 1; j++) {
            for (auto& str : datas) {
                char_string_reader->init(str.data(), str.size(), false);
                auto stream = analyzer->reusableTokenStream(field->name(), char_string_reader);
                field->setValue(stream);

                // field->setValue(str.data(), str.size());

                indexwriter->addDocument(doc);
            }
        }
        indexwriter->close();

        _CLLDELETE(indexwriter);
        _CLLDELETE(doc);
        _CLLDELETE(analyzer);
        _CLLDELETE(char_string_reader);

        auto ret = index_file_writer->begin_close();
        if (!ret.ok()) {
            std::cerr << "IndexFileWriter close error:" << ret.msg() << std::endl;
            return -1;
        }
        ret = index_file_writer->finish_close();
        if (!ret.ok()) {
            std::cerr << "IndexFileWriter wait close error:" << ret.msg() << std::endl;
            return -1;
        }
    } else if (FLAGS_operation == "show_nested_files_v2") {
        if (FLAGS_idx_file_path == "") {
            std::cout << "no file flag for show " << std::endl;
            return -1;
        }
        std::string index_path_prefix =
                google::protobuf::StripSuffixString(FLAGS_idx_file_path, ".idx");
        auto fs = doris::io::global_local_filesystem();
        try {
            auto index_file_reader = std::make_unique<IndexFileReader>(
                    fs, index_path_prefix, doris::InvertedIndexStorageFormatPB::V2);
            auto st = index_file_reader->init(4096);
            if (!st.ok()) {
                std::cerr << "IndexFileReader init error:" << st.msg() << std::endl;
                return -1;
            }
            std::cout << "Nested files for " << index_path_prefix << std::endl;
            std::cout << "==================================" << std::endl;
            auto dirs = index_file_reader->get_all_directories();
            for (auto& dir : *dirs) {
                auto index_id = dir.first.first;
                auto index_suffix = dir.first.second;

                if (FLAGS_idx_id > 0 && index_id != FLAGS_idx_id) {
                    continue;
                }

                std::vector<std::string> files;
                doris::TabletIndexPB index_pb;
                index_pb.set_index_id(index_id);
                index_pb.set_index_suffix_name(index_suffix);
                TabletIndex index_meta;
                index_meta.init_from_pb(index_pb);
                std::cout << "index_id:" << index_id << " index_suffix:" << index_suffix
                          << std::endl;

                CLuceneError err;
                auto ret = index_file_reader->open(&index_meta);
                if (!ret.has_value()) {
                    std::cerr << "IndexFileReader open error:" << ret.error() << std::endl;
                    return -1;
                }
                using T = std::decay_t<decltype(ret)>;
                auto reader = std::forward<T>(ret).value();
                reader->list(&files);
                for (auto& file : files) {
                    int64_t size = reader->fileLength(file.c_str());
                    std::cout << file << "\t" << size << " bytes" << std::endl;
                }
            }
        } catch (CLuceneError& err) {
            std::cerr << "error occurred when show files: " << err.what() << std::endl;
        }
    } else if (FLAGS_operation == "check_terms_stats_v2") {
        if (FLAGS_idx_file_path == "" || FLAGS_idx_id <= 0) {
            std::cout << "no file flag for check " << std::endl;
            return -1;
        }
        std::string index_path_prefix =
                google::protobuf::StripSuffixString(FLAGS_idx_file_path, ".idx");
        auto fs = doris::io::global_local_filesystem();
        try {
            auto index_file_reader = std::make_unique<IndexFileReader>(
                    fs, index_path_prefix, doris::InvertedIndexStorageFormatPB::V2);
            auto st = index_file_reader->init(4096);
            if (!st.ok()) {
                std::cerr << "IndexFileReader init error:" << st.msg() << std::endl;
                return -1;
            }
            std::vector<std::string> files;
            int64_t index_id = FLAGS_idx_id;
            std::string index_suffix = "";
            doris::TabletIndexPB index_pb;
            index_pb.set_index_id(index_id);
            index_pb.set_index_suffix_name(index_suffix);
            TabletIndex index_meta;
            index_meta.init_from_pb(index_pb);
            auto ret = index_file_reader->open(&index_meta);
            if (!ret.has_value()) {
                std::cerr << "IndexFileReader open error:" << ret.error() << std::endl;
                return -1;
            }
            using T = std::decay_t<decltype(ret)>;
            auto reader = std::forward<T>(ret).value();
            index_file_reader->debug_file_entries();
            std::cout << "Term statistics for " << index_path_prefix << std::endl;
            std::cout << "==================================" << std::endl;
            check_terms_stats(reader.get());
            reader->close();
        } catch (CLuceneError& err) {
            std::cerr << "error occurred when check_terms_stats: " << err.what() << std::endl;
        }
    } else {
        std::cerr << "invalid operation: " << FLAGS_operation << "\n" << usage << std::endl;
        return -1;
    }
    gflags::ShutDownCommandLineFlags();
    return 0;
}