mcm-26-b/scripts/benchmark_q3_search.py

"""Offline Q3 search ablations for docs/algo-3.md; no official socket access.

The production planner is unchanged. Only coverage and task scheduling vary.
Run: python scripts/benchmark_q3_search.py --seeds 0,1,2,3,4
"""
from __future__ import annotations

import argparse
from dataclasses import asdict
import hashlib
import json
import math
from pathlib import Path
from statistics import mean, median
from time import perf_counter

from mcm26b.belief import ChannelStatus  # noqa: F401  (兼容旧 import 路径)
from mcm26b.config import ProblemConfig
from mcm26b.coverage import CoverageLedger  # noqa: F401
from mcm26b.planner import OmnidirectionalLoop, clear_certificate  # noqa: F401
from mcm26b.planner.seven import (SevenBatch, SevenImmediate, SevenSweep,
                                  RING_RADIUS_M, seven_points)
from mcm26b.protocol.client import SimulatorClient
from mcm26b.protocol.rules import NEAR  # noqa: F401
from mcm26b.sim import LocalSimulator, LocalSimTransport, Source
from mcm26b.sim.scenes import random_omni_scene

ROOT = Path(__file__).resolve().parents[1]


VARIANTS = {"grid49_immediate": OmnidirectionalLoop, "seven_immediate": SevenImmediate,
            "seven_batch": SevenBatch, "seven_sweep": SevenSweep}


def run_case(name, sources, error_seed, out):
    scene_path = out / "scenes" / f"{name}.json"
    scene_path.parent.mkdir(parents=True, exist_ok=True)
    scene_path.write_text(json.dumps({"sources": [asdict(s) for s in sources],
                                    "error_seed": error_seed}, indent=2), encoding="utf-8")
    config = ProblemConfig()
    rows = []
    for label, cls in VARIANTS.items():
        sim = LocalSimulator(sources, robot_id="Q3-ABLATION", config=config, error_seed=error_seed)
        client = SimulatorClient("Q3-ABLATION", config=config, transport=LocalSimTransport(sim))
        loop = cls(client, config)
        start = perf_counter()
        summary = loop.run()
        elapsed = perf_counter() - start
        actions = client.log.action_events()
        log_path = out / "actions" / f"{name}-{label}.jsonl"
        log_path.parent.mkdir(parents=True, exist_ok=True)
        with log_path.open("w", encoding="utf-8") as handle:
            for event in actions:
                handle.write(json.dumps(event, ensure_ascii=False) + "\n")
        truth = {s.channel for s in sources}
        absent = set(range(1, 21)) - truth
        absence_valid = all(loop.ledger.coverage_complete(c) for c in summary["absent_channels"])
        row = {"case": name, "variant": label, "source_count": len(sources),
               "all_cleared": sim.cleared == truth, "completed": summary["completed"],
               "correct_absence": set(summary["absent_channels"]) <= absent and absence_valid,
               "normal_exit": summary["exit_reason"] == "user_exit",
               "virtual_time_s": sim.virtual_time_s, "measures": summary["measures"],
               "clears": summary["clears"], "failed_clears": summary["clears"]-len(sim.cleared),
               "distance_m": summary["move_distance_m"], "compute_wall_s": elapsed,
               "anomalies": summary["anomalies"], "stop_reason": summary["stop_reason"],
               "last_clear_s": max((e["response"]["virtual_time_s"] for e in actions
                                    if e["path"] == "/clear" and
                                    e["response"].get("clear_result") == "success"), default=0),
               "actions_file": str(log_path.relative_to(out)),
               "actions_sha256": hashlib.sha256(log_path.read_bytes()).hexdigest()}
        rows.append(row)
        print(f"{name} {label}: all_cleared={row['all_cleared']} T={row['virtual_time_s']:.1f}s M={row['measures']}", flush=True)
    return rows


def geometry_check():
    points = seven_points()
    field = ProblemConfig().field
    # Exact radial-sector argument: switch nearest site at r=a/sqrt(3).
    crossover = RING_RADIUS_M / math.sqrt(3)
    boundary = math.sqrt(field.omega_radius_m**2 + RING_RADIUS_M**2
                         - math.sqrt(3)*field.omega_radius_m*RING_RADIUS_M)
    analytic = max(crossover, boundary)
    worst = (0.0, None)
    for ir in range(181):
        r = field.omega_radius_m * ir / 180
        for ia in range(721):
            angle = 2 * math.pi * ia / 720
            p = (r*math.cos(angle), r*math.sin(angle))
            d = min(math.dist(p, s) for s in points)
            if d > worst[0]:
                worst = (d, p)
    return {"ring_radius_m": RING_RADIUS_M, "points": points,
            "analytic_cover_radius_m": analytic, "reception_margin_m": field.r_eff_min_m-analytic,
            "sampled_max_distance_m": worst[0], "sampled_worst_point": worst[1],
            "samples": 181*721,
            "passed": worst[0] <= analytic+1e-8 and analytic < field.r_eff_min_m,
            "note": "Grid samples check the derivation; the sector proof establishes continuous coverage."}


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--seeds", default="0,1,2,3,4")
    parser.add_argument("--out", type=Path, default=ROOT/"output/q3/search-ablation")
    args = parser.parse_args()
    args.out.mkdir(parents=True, exist_ok=True)
    rows = []
    for count in (10, 16):
        for seed in [int(s) for s in args.seeds.split(",")]:
            rows.extend(run_case(f"random-{count}-seed{seed}", random_omni_scene(seed, count),
                                 20260910+seed, args.out))
        # Sources on the exact outer boundary at sector seams; minimum receiving radius.
        edge = [Source(c+1, 1800*math.cos(math.pi/6+2*math.pi*c/count),
                       1800*math.sin(math.pi/6+2*math.pi*c/count), 1000) for c in range(count)]
        rows.extend(run_case(f"boundary-{count}", edge, 20260910, args.out))
    aggregates = {}
    for label in VARIANTS:
        part = [r for r in rows if r["variant"] == label]
        baseline = {r["case"]: r for r in rows if r["variant"] == "grid49_immediate"}
        savings = [1-r["virtual_time_s"]/baseline[r["case"]]["virtual_time_s"] for r in part]
        aggregates[label] = {
            "runs": len(part), "all_cleared_runs": sum(r["all_cleared"] for r in part),
            "valid_runs": sum(r["all_cleared"] and r["completed"] and r["correct_absence"]
                              and r["normal_exit"] and not r["anomalies"] for r in part),
            "mean_virtual_s": mean(r["virtual_time_s"] for r in part),
            "median_virtual_s": median(r["virtual_time_s"] for r in part),
            "max_virtual_s": max(r["virtual_time_s"] for r in part),
            "mean_measures": mean(r["measures"] for r in part),
            "mean_distance_m": mean(r["distance_m"] for r in part),
            "mean_compute_wall_s": mean(r["compute_wall_s"] for r in part),
            "mean_paired_time_reduction": mean(savings),
            "min_paired_time_reduction": min(savings),
            "max_paired_time_reduction": max(savings),
            "failed_clears": sum(r["failed_clears"] for r in part),
        }
    result = {"source": "local_simulator_only", "production_algorithm_modified": False,
              "seeds": args.seeds, "config": asdict(ProblemConfig()),
              "geometry": geometry_check(), "aggregates": aggregates, "runs": rows,
              "limitations": "Same scene and fixed per-position error field for each variant; synthetic data, not an official improvement estimate. Only search/scheduling is changed; no posterior or localizer redesign."}
    (args.out/"benchmark.json").write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
    print(json.dumps(aggregates, ensure_ascii=False, indent=2))
    return 0 if result["geometry"]["passed"] and all(a["valid_runs"] == a["runs"] for a in aggregates.values()) else 1


if __name__ == "__main__":
    raise SystemExit(main())