cpp/experiment/read_memory/read_mem_model.cpp

/*
 * Read-path memory model verification experiment.
 *
 * Validates the memory formula:
 *   M_read ≈ M_fixed + batch_size × s_row + N_cols × C_page
 *
 * Experiments with different column counts and batch sizes,
 * recording peak memory per ModStat for each configuration.
 *
 * Prerequisite: run no_flush_bench (write_memory) first to generate
 * experiment.tsfile.
 *
 * Build:
 *   cmake -DENABLE_MEM_STAT=ON -DBUILD_TEST=OFF ..
 *   cmake --build . --target read_mem_model
 */

#include <sys/resource.h>
#include <sys/time.h>

#include <chrono>
#include <cstdio>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <string>
#include <vector>

#include "common/allocator/alloc_base.h"
#include "reader/table_result_set.h"
#include "reader/tsfile_reader.h"

static const char* kTable = "mem_bench";

static const int kModCount = common::__LAST_MOD_ID;

static int64_t tv_to_us(const struct timeval& tv) {
    return static_cast<int64_t>(tv.tv_sec) * 1000000 + tv.tv_usec;
}

using steady_clock = std::chrono::steady_clock;
static steady_clock::time_point g_t0;

static void write_csv_header(std::ofstream& csv) {
    csv << "n_cols,batch_size,rows_read,phase,wall_us,user_cpu_us,sys_cpu_us";
    for (int i = 0; i < kModCount; i++) csv << "," << common::g_mod_names[i];
    csv << ",TOTAL\n";
}

static void write_csv_row(std::ofstream& csv, int n_cols, uint32_t batch_size,
                          int64_t rows, const char* phase) {
    int64_t wall_us = std::chrono::duration_cast<std::chrono::microseconds>(
                          steady_clock::now() - g_t0)
                          .count();
    struct rusage ru;
    getrusage(RUSAGE_SELF, &ru);
    int64_t user_us = tv_to_us(ru.ru_utime);
    int64_t sys_us = tv_to_us(ru.ru_stime);

    auto& ms = common::ModStat::get_instance();
    csv << n_cols << "," << batch_size << "," << rows << "," << phase << ","
        << wall_us << "," << user_us << "," << sys_us;
    int64_t total = 0;
    for (int i = 0; i < kModCount; i++) {
        int32_t val = ms.get_stat(i);
        csv << "," << val;
        total += val;
    }
    csv << "," << total << "\n";
}

static void print_separator(const char* label) {
    std::cout << "\n──── " << label << " ";
    for (int i = 0; i < 50 - (int)std::string(label).size(); i++)
        std::cout << "─";
    std::cout << "\n";
}

int main() {
    // Path to tsfile generated by no_flush_bench (write_memory experiment).
    static const char* baseline_path =
        "/Users/colin/dev/tsfile_b1/cpp/experiment/experiment.tsfile";
    static const char* csv_prefix = "read_memory_results";
    static const int64_t baseline_rows = 200000000;

    std::cout << "=== Read Memory Model Experiment ===\n"
              << "  baseline:    " << baseline_path << "\n"
              << "  csv_prefix:  " << csv_prefix << "\n\n";

    static const int64_t print_interval_rows = 1000000;  // sample every 1M rows

    storage::libtsfile_init();
    g_t0 = steady_clock::now();

    // Parameter matrix
    int col_counts[] = {1, 2, 4, 6};
    uint32_t batch_sizes[] = {1024, 4096, 16384, 65536};

    print_separator("BASELINE");
    std::cout << "Baseline file: " << baseline_path << "\n";

    print_separator("EXPERIMENTS");
    std::cout << "Parameter matrix: N_cols=" << col_counts[0];
    for (size_t i = 1; i < sizeof(col_counts) / sizeof(col_counts[0]); i++)
        std::cout << "/" << col_counts[i];
    std::cout << ", batch_size=";
    std::cout << batch_sizes[0];
    for (size_t i = 1; i < sizeof(batch_sizes) / sizeof(batch_sizes[0]); i++)
        std::cout << "/" << batch_sizes[i];
    std::cout << "\n\n";

    int exp_count = 0;
    for (int n_cols : col_counts) {
        for (uint32_t batch_size : batch_sizes) {
            // Generate CSV filename with parameters
            char csv_filename[256];
            snprintf(csv_filename, sizeof(csv_filename),
                     "%s_cols%d_batch%u.csv", csv_prefix, n_cols, batch_size);

            std::ofstream csv(csv_filename);
            if (!csv.is_open()) {
                std::cerr << "cannot open csv: " << csv_filename << "\n";
                return 1;
            }

            write_csv_header(csv);

            // Select column names (first n_cols FIELD columns)
            static const char* kFieldCols[] = {"s1", "s2", "s3", "s4",
                                               "s5", "s6", "s7", "s8"};
            std::vector<std::string> cols = {"id1", "id2"};
            for (int c = 0; c < n_cols && c < 8; c++)
                cols.push_back(kFieldCols[c]);

            // Open reader and query
            storage::TsFileReader reader;
            std::cout << "  [open]  cols=" << n_cols << " batch=" << batch_size
                      << std::flush;
            if (reader.open(baseline_path) != 0) {
                std::cerr << "\nFailed to open: " << baseline_path << "\n";
                return 1;
            }
            std::cout << " [query]" << std::flush;
            storage::ResultSet* rs = nullptr;
            int ret =
                reader.query(kTable, cols, 0, baseline_rows, rs, batch_size);
            if (ret != 0) {
                std::cerr << "\nquery failed: " << ret << "\n";
                reader.close();
                return 1;
            }
            if (rs == nullptr) {
                std::cerr << "\nresult_set is null after query\n";
                reader.close();
                return 1;
            }
            std::cout << " [read]" << std::flush;

            common::TsBlock* block = nullptr;
            int64_t total_rows_read = 0;
            int64_t next_sample_at = print_interval_rows;

            write_csv_row(csv, n_cols, batch_size, 0, "before_read");

            while (rs->get_next_tsblock(block) == common::E_OK && block) {
                total_rows_read += block->get_row_count();
                while (total_rows_read >= next_sample_at) {
                    write_csv_row(csv, n_cols, batch_size, total_rows_read,
                                  "read");
                    next_sample_at += print_interval_rows;
                }
            }

            write_csv_row(csv, n_cols, batch_size, total_rows_read,
                          "after_read");

            rs->close();
            delete rs;
            reader.close();
            csv.close();

            std::cout << "  [" << std::setw(2) << (++exp_count) << "] "
                      << "cols=" << std::setw(2) << n_cols
                      << " batch=" << std::setw(5) << batch_size
                      << " rows_read=" << total_rows_read << " -> "
                      << csv_filename << "\n";
        }
    }

    print_separator("SUMMARY");
    std::cout << "Total experiments: " << exp_count << "\n"
              << "CSV files: " << csv_prefix
              << "_cols{1,4,8,16}_batch{1024,4096,16384,65536}.csv\n";

    storage::libtsfile_destroy();
    return 0;
}