benchmarks/csharp/benchmark_report.py

#!/usr/bin/env python3
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import argparse
import json
import os
import platform
import shutil
import subprocess
from collections import defaultdict
from datetime import datetime

import matplotlib.pyplot as plt
import numpy as np

try:
    import psutil

    HAS_PSUTIL = True
except ImportError:
    HAS_PSUTIL = False

COLORS = {
    "fory": "#FF6f01",
    "protobuf": "#55BCC2",
    "msgpack": (0.55, 0.40, 0.45),
}
SERIALIZER_ORDER = ["fory", "protobuf", "msgpack"]
SERIALIZER_LABELS = {
    "fory": "fory",
    "protobuf": "protobuf",
    "msgpack": "msgpack",
}
PREFERRED_DATATYPE_ORDER = [
    "struct",
    "sample",
    "mediacontent",
    "structlist",
    "samplelist",
    "mediacontentlist",
]
PREFERRED_OPERATION_ORDER = ["serialize", "deserialize"]


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description="Plot C# benchmark stats and generate Markdown report"
    )
    parser.add_argument(
        "--json-file",
        default="benchmark_results.json",
        help="Benchmark JSON output file",
    )
    parser.add_argument(
        "--output-dir",
        default="",
        help="Output directory for plots and report",
    )
    parser.add_argument(
        "--plot-prefix",
        default="",
        help="Image path prefix in Markdown report",
    )
    return parser.parse_args()


def load_results(path: str) -> dict:
    with open(path, "r", encoding="utf-8") as f:
        return json.load(f)


def get_system_info(benchmark_data: dict) -> dict:
    info = {
        "OS": benchmark_data.get(
            "OsDescription", f"{platform.system()} {platform.release()}"
        ),
        "OS Architecture": benchmark_data.get("OsArchitecture", "Unknown"),
        "Machine": benchmark_data.get("ProcessArchitecture", platform.machine()),
        "Runtime Version": benchmark_data.get("RuntimeVersion", "Unknown"),
        "Benchmark Date (UTC)": benchmark_data.get("GeneratedAtUtc", "Unknown"),
        "Warmup Seconds": benchmark_data.get("WarmupSeconds", "Unknown"),
        "Duration Seconds": benchmark_data.get("DurationSeconds", "Unknown"),
    }
    processor_count = benchmark_data.get("ProcessorCount")
    if processor_count is not None:
        info["CPU Logical Cores (from benchmark)"] = processor_count

    if HAS_PSUTIL:
        info["CPU Cores (Physical)"] = psutil.cpu_count(logical=False)
        info["CPU Cores (Logical)"] = psutil.cpu_count(logical=True)
        info["Total RAM (GB)"] = round(psutil.virtual_memory().total / (1024**3), 2)
    return info


def format_datatype_label(datatype: str) -> str:
    if datatype.endswith("list"):
        base = datatype[: -len("list")]
        if base == "mediacontent":
            return "MediaContent\nList"
        return f"{base.capitalize()}\nList"
    if datatype == "mediacontent":
        return "MediaContent"
    return datatype.capitalize()


def format_datatype_table_label(datatype: str) -> str:
    if datatype.endswith("list"):
        base = datatype[: -len("list")]
        if base == "mediacontent":
            return "MediaContentList"
        return f"{base.capitalize()}List"
    if datatype == "mediacontent":
        return "MediaContent"
    return datatype.capitalize()


def format_tps_label(tps: float) -> str:
    if tps >= 1e9:
        return f"{tps / 1e9:.2f}G"
    if tps >= 1e6:
        return f"{tps / 1e6:.2f}M"
    if tps >= 1e3:
        return f"{tps / 1e3:.2f}K"
    return f"{tps:.0f}"


def preferred_ordered_values(values, preferred):
    return [item for item in preferred if item in values] + sorted(
        item for item in values if item not in preferred
    )


def process_benchmark_rows(rows):
    raw_timings = defaultdict(lambda: defaultdict(lambda: defaultdict(list)))
    raw_throughputs = defaultdict(lambda: defaultdict(lambda: defaultdict(list)))
    raw_sizes = defaultdict(lambda: defaultdict(list))

    for row in rows:
        serializer = str(row.get("Serializer", "")).lower()
        data_type = str(row.get("DataType", "")).lower()
        operation = str(row.get("Operation", "")).lower()
        if not serializer or not data_type or not operation:
            continue

        avg_ns_raw = row.get("AverageNanoseconds")
        ops_raw = row.get("OperationsPerSecond")
        avg_ns = float(avg_ns_raw) if avg_ns_raw is not None else 0.0
        ops = float(ops_raw) if ops_raw is not None else 0.0

        if avg_ns <= 0.0 and ops > 0.0:
            avg_ns = 1e9 / ops
        if ops <= 0.0 and avg_ns > 0.0:
            ops = 1e9 / avg_ns

        if avg_ns > 0.0:
            raw_timings[data_type][operation][serializer].append(avg_ns)
        if ops > 0.0:
            raw_throughputs[data_type][operation][serializer].append(ops)

        serialized_size = row.get("SerializedSize")
        if serialized_size is not None:
            raw_sizes[data_type][serializer].append(int(round(serialized_size)))

    timings = defaultdict(lambda: defaultdict(dict))
    throughputs = defaultdict(lambda: defaultdict(dict))
    sizes = defaultdict(dict)

    for data_type, op_values in raw_timings.items():
        for operation, serializer_values in op_values.items():
            for serializer, values in serializer_values.items():
                timings[data_type][operation][serializer] = sum(values) / len(values)

    for data_type, op_values in raw_throughputs.items():
        for operation, serializer_values in op_values.items():
            for serializer, values in serializer_values.items():
                throughputs[data_type][operation][serializer] = sum(values) / len(
                    values
                )

    for data_type, serializer_values in raw_sizes.items():
        for serializer, values in serializer_values.items():
            sizes[data_type][serializer] = sum(values) / len(values)

    return timings, throughputs, sizes


def build_coverage(rows):
    cases = set()
    serializers = set()
    datatypes = set()
    operations = set()
    for row in rows:
        serializer = str(row.get("Serializer", "")).lower()
        data_type = str(row.get("DataType", "")).lower()
        operation = str(row.get("Operation", "")).lower()
        if not serializer or not data_type or not operation:
            continue
        cases.add((serializer, data_type, operation))
        serializers.add(serializer)
        datatypes.add(data_type)
        operations.add(operation)

    expected_cases = (
        len(SERIALIZER_ORDER)
        * len(PREFERRED_DATATYPE_ORDER)
        * len(PREFERRED_OPERATION_ORDER)
    )
    return {
        "case_count": len(cases),
        "expected_case_count": expected_cases,
        "serializers": sorted(serializers),
        "datatypes": preferred_ordered_values(
            list(datatypes), PREFERRED_DATATYPE_ORDER
        ),
        "operations": preferred_ordered_values(
            list(operations), PREFERRED_OPERATION_ORDER
        ),
        "is_partial": len(cases) < expected_cases,
    }


def plot_datatype(ax, throughputs: dict, datatype: str, operation: str) -> None:
    if datatype not in throughputs or operation not in throughputs[datatype]:
        ax.set_title(f"{datatype} {operation} - No Data")
        ax.axis("off")
        return

    libs = set(throughputs[datatype][operation].keys())
    lib_order = [lib for lib in SERIALIZER_ORDER if lib in libs]
    if not lib_order:
        ax.set_title(f"{datatype} {operation} - No Supported Serializer Data")
        ax.axis("off")
        return
    throughput = [throughputs[datatype][operation].get(lib, 0) for lib in lib_order]
    colors = [COLORS.get(lib, "#888888") for lib in lib_order]

    x = np.arange(len(lib_order))
    bars = ax.bar(x, throughput, color=colors, width=0.6)

    ax.set_title(f"{operation.capitalize()} Throughput (higher is better)")
    ax.set_xticks(x)
    ax.set_xticklabels([SERIALIZER_LABELS.get(lib, lib) for lib in lib_order])
    ax.set_ylabel("Throughput (ops/sec)")
    ax.grid(True, axis="y", linestyle="--", alpha=0.5)
    ax.ticklabel_format(style="scientific", axis="y", scilimits=(0, 0))

    for bar, tps_value in zip(bars, throughput):
        height = bar.get_height()
        ax.annotate(
            format_tps_label(tps_value),
            xy=(bar.get_x() + bar.get_width() / 2, height),
            xytext=(0, 3),
            textcoords="offset points",
            ha="center",
            va="bottom",
            fontsize=9,
        )


def plot_combined_tps_subplot(ax, throughputs, grouped_datatypes, operation, title):
    if not grouped_datatypes:
        ax.set_title(f"{title}\nNo Data")
        ax.axis("off")
        return

    x = np.arange(len(grouped_datatypes))
    available_libs = [
        lib
        for lib in SERIALIZER_ORDER
        if any(throughputs[dt][operation].get(lib, 0) > 0 for dt in grouped_datatypes)
    ]
    if not available_libs:
        ax.set_title(f"{title}\nNo Data")
        ax.axis("off")
        return

    width = 0.8 / len(available_libs)
    for idx, lib in enumerate(available_libs):
        tps = [throughputs[dt][operation].get(lib, 0) for dt in grouped_datatypes]
        offset = (idx - (len(available_libs) - 1) / 2) * width
        ax.bar(
            x + offset,
            tps,
            width,
            label=SERIALIZER_LABELS.get(lib, lib),
            color=COLORS.get(lib, "#888888"),
        )

    ax.set_title(title)
    ax.set_xticks(x)
    ax.set_xticklabels([format_datatype_label(dt) for dt in grouped_datatypes])
    ax.legend()
    ax.grid(True, axis="y", linestyle="--", alpha=0.5)
    ax.ticklabel_format(style="scientific", axis="y", scilimits=(0, 0))


def build_markdown(
    args,
    system_info,
    coverage,
    timings,
    throughputs,
    sizes,
    datatypes,
    operations,
    plot_images,
):
    report_lines = [
        "# C# Benchmark Performance Report\n\n",
        f"_Generated on {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}_\n\n",
        "## How to Generate This Report\n\n",
        "```bash\n",
        "cd benchmarks/csharp\n",
        "dotnet run -c Release --project ./Fory.CSharpBenchmark.csproj -- --output build/benchmark_results.json\n",
        "python3 benchmark_report.py --json-file build/benchmark_results.json --output-dir report\n",
        "```\n\n",
        "## Hardware & OS Info\n\n",
        "| Key | Value |\n",
        "|-----|-------|\n",
    ]

    for key, value in system_info.items():
        report_lines.append(f"| {key} | {value} |\n")

    report_lines.append("\n## Benchmark Coverage\n\n")
    report_lines.append("| Key | Value |\n")
    report_lines.append("|-----|-------|\n")
    report_lines.append(
        f"| Cases in input JSON | {coverage['case_count']} / {coverage['expected_case_count']} |\n"
    )
    report_lines.append(
        f"| Serializers | {', '.join(coverage['serializers']) or 'N/A'} |\n"
    )
    report_lines.append(
        f"| Datatypes | {', '.join(coverage['datatypes']) or 'N/A'} |\n"
    )
    report_lines.append(
        f"| Operations | {', '.join(coverage['operations']) or 'N/A'} |\n"
    )
    if coverage["is_partial"]:
        report_lines.append(
            "\n> Warning: benchmark input is partial; plots/tables only show available cases.\n"
        )

    report_lines.append("\n## Benchmark Plots\n")
    report_lines.append("\nAll class-level plots below show throughput (ops/sec).\n")
    plot_images_sorted = sorted(
        plot_images, key=lambda item: (0 if item[0] == "throughput" else 1, item[0])
    )
    for datatype, img in plot_images_sorted:
        img_filename = os.path.basename(img)
        img_path_report = args.plot_prefix + img_filename
        plot_title = datatype.replace("_", " ").title()
        report_lines.append(f"\n### {plot_title}\n\n")
        report_lines.append(f"![{plot_title}]({img_path_report})\n")

    report_lines.append("\n## Benchmark Results\n\n")
    report_lines.append("### Timing Results (nanoseconds)\n\n")
    report_lines.append(
        "| Datatype | Operation | fory (ns) | protobuf (ns) | msgpack (ns) | Fastest |\n"
    )
    report_lines.append(
        "|----------|-----------|-----------|---------------|--------------|---------|\n"
    )

    for datatype in datatypes:
        for operation in operations:
            times = {
                lib: timings[datatype][operation].get(lib, 0)
                for lib in SERIALIZER_ORDER
            }
            positive_times = {lib: t for lib, t in times.items() if t > 0}
            fastest = "N/A"
            if positive_times:
                fastest_lib = min(positive_times, key=positive_times.get)
                fastest = SERIALIZER_LABELS.get(fastest_lib, fastest_lib)
            report_lines.append(
                "| "
                + f"{format_datatype_table_label(datatype)} | {operation.capitalize()} | "
                + " | ".join(
                    f"{times[lib]:.1f}" if times[lib] > 0 else "N/A"
                    for lib in SERIALIZER_ORDER
                )
                + f" | {fastest} |\n"
            )

    report_lines.append("\n### Throughput Results (ops/sec)\n\n")
    report_lines.append(
        "| Datatype | Operation | fory TPS | protobuf TPS | msgpack TPS | Fastest |\n"
    )
    report_lines.append(
        "|----------|-----------|----------|--------------|-------------|---------|\n"
    )

    for datatype in datatypes:
        for operation in operations:
            tps_values = {
                lib: throughputs[datatype][operation].get(lib, 0)
                for lib in SERIALIZER_ORDER
            }
            positive_tps = {lib: v for lib, v in tps_values.items() if v > 0}
            fastest = "N/A"
            if positive_tps:
                fastest_lib = max(positive_tps, key=positive_tps.get)
                fastest = SERIALIZER_LABELS.get(fastest_lib, fastest_lib)
            report_lines.append(
                "| "
                + f"{format_datatype_table_label(datatype)} | {operation.capitalize()} | "
                + " | ".join(
                    f"{tps_values[lib]:,.0f}" if tps_values[lib] > 0 else "N/A"
                    for lib in SERIALIZER_ORDER
                )
                + f" | {fastest} |\n"
            )

    if sizes:
        report_lines.append("\n### Serialized Data Sizes (bytes)\n\n")
        report_lines.append("| Datatype | fory | protobuf | msgpack |\n")
        report_lines.append("|----------|------|----------|---------|\n")
        for datatype in datatypes:
            row_values = []
            has_value = False
            for serializer in SERIALIZER_ORDER:
                size = sizes[datatype].get(serializer)
                if size is None:
                    row_values.append("N/A")
                else:
                    row_values.append(str(int(round(size))))
                    has_value = True
            if has_value:
                report_lines.append(
                    f"| {format_datatype_table_label(datatype)} | "
                    + " | ".join(row_values)
                    + " |\n"
                )

    return "".join(report_lines)


def main() -> None:
    args = parse_args()
    benchmark_data = load_results(args.json_file)

    if args.output_dir.strip():
        output_dir = args.output_dir
    else:
        output_dir = datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
    os.makedirs(output_dir, exist_ok=True)

    rows = benchmark_data.get("Results", [])
    timings, throughputs, sizes = process_benchmark_rows(rows)
    coverage = build_coverage(rows)

    datatype_candidates = (
        set(timings.keys()) | set(throughputs.keys()) | set(sizes.keys())
    )
    datatypes = preferred_ordered_values(
        list(datatype_candidates), PREFERRED_DATATYPE_ORDER
    )
    operations_present = set()
    for datatype in datatypes:
        operations_present.update(timings[datatype].keys())
        operations_present.update(throughputs[datatype].keys())
    operations = preferred_ordered_values(
        list(operations_present), PREFERRED_OPERATION_ORDER
    )

    plot_images = []
    for datatype in datatypes:
        fig, axes = plt.subplots(1, 2, figsize=(12, 5))
        for index, operation in enumerate(PREFERRED_OPERATION_ORDER):
            plot_datatype(axes[index], throughputs, datatype, operation)
        fig.suptitle(f"{format_datatype_table_label(datatype)} Throughput", fontsize=14)
        fig.tight_layout(rect=[0, 0, 1, 0.95])
        plot_path = os.path.join(output_dir, f"{datatype}.png")
        plt.savefig(plot_path, dpi=150)
        plot_images.append((datatype, plot_path))
        plt.close()

    non_list_datatypes = [dt for dt in datatypes if not dt.endswith("list")]
    list_datatypes = [dt for dt in datatypes if dt.endswith("list")]
    fig, axes = plt.subplots(1, 4, figsize=(28, 6))
    combined_subplots = [
        (
            axes[0],
            non_list_datatypes,
            "serialize",
            "Serialize Throughput (higher is better)",
        ),
        (
            axes[1],
            non_list_datatypes,
            "deserialize",
            "Deserialize Throughput (higher is better)",
        ),
        (
            axes[2],
            list_datatypes,
            "serialize",
            "Serialize Throughput (*List, higher is better)",
        ),
        (
            axes[3],
            list_datatypes,
            "deserialize",
            "Deserialize Throughput (*List, higher is better)",
        ),
    ]
    for ax, grouped_datatypes, operation, title in combined_subplots:
        plot_combined_tps_subplot(ax, throughputs, grouped_datatypes, operation, title)
    # Keep the y-label style consistent with single-datatype plots (axis label,
    # not figure-level label) and avoid overlap with y-ticks.
    axes[0].set_ylabel("Throughput (ops/sec)")
    fig.tight_layout()
    throughput_path = os.path.join(output_dir, "throughput.png")
    plt.savefig(throughput_path, dpi=150)
    plot_images.append(("throughput", throughput_path))
    plt.close()

    report = build_markdown(
        args=args,
        system_info=get_system_info(benchmark_data),
        coverage=coverage,
        timings=timings,
        throughputs=throughputs,
        sizes=sizes,
        datatypes=datatypes,
        operations=operations,
        plot_images=plot_images,
    )
    legacy_report_path = os.path.join(output_dir, "REPORT.md")
    if os.path.exists(legacy_report_path):
        os.remove(legacy_report_path)
    report_path = os.path.join(output_dir, "README.md")
    with open(report_path, "w", encoding="utf-8") as f:
        f.write(report)

    prettier = shutil.which("prettier")
    if prettier is not None:
        subprocess.run([prettier, "--write", report_path], check=True)

    if coverage["is_partial"]:
        print(
            "Warning: partial benchmark input detected "
            f"({coverage['case_count']}/{coverage['expected_case_count']} cases)."
        )
    print(f"Plots saved in: {output_dir}")
    print(f"Markdown report generated at: {report_path}")


if __name__ == "__main__":
    main()