baseline

2026-09-12 16:58:25 +08:00
parent 1fa2264a17
commit 892451dca9
36 changed files with 8767 additions and 0 deletions
@@ -0,0 +1,308 @@
+"""对比框架公共层：方法描述、指标聚合、结果落盘、离线闭环装配。
+
+设计口径
+--------
+- **最优实现** = 复用 ``src/mcm26b`` 中已由测试固化的生产实现；
+- **基线实现** = 同题下更朴素但可独立运行的参照实现（不得共享最优实现的
+  关键步骤），用于量化"生产实现好在哪里、差多少"；
+- **参考解** = 与分析算法相互独立的数值/枚举标准，用来判定谁对谁错。
+
+第三、四问是闭环问题，统一走 ``LocalSimulator`` 内存回环（与官方 HTTP
+协议同一边界），真值只在评测机侧，策略端只看响应。
+"""
+
+from __future__ import annotations
+
+import json
+import statistics
+from dataclasses import dataclass, replace
+from pathlib import Path
+from time import perf_counter
+
+from mcm26b.config import ProblemConfig
+from mcm26b.evaluation import recompute_virtual_time
+from mcm26b.harness import LocalSimTransport, LocalSimulator
+from mcm26b.protocol.client import SimulatorClient
+
+ROOT = Path(__file__).resolve().parents[1]
+RESULTS_DIR = Path(__file__).resolve().parent / "results"
+
+
+@dataclass(frozen=True)
+class Method:
+    """一个可对比方法：key 用作图表列名，kind 用于区分最优/基线/参考。"""
+
+    key: str
+    name: str
+    kind: str          # optimal | baseline | reference
+    note: str = ""
+
+
+def problem_config(*, persistent_search: bool = False,
+                   margin_deg: float | None = None) -> ProblemConfig:
+    """对比口径配置：默认关闭"扫到限时"的持续复扫（对齐首次找全证明）。"""
+    base = ProblemConfig()
+    kwargs: dict = {"persistent_search": persistent_search}
+    if margin_deg is not None:
+        kwargs["bearing_rounding_margin_deg"] = margin_deg
+    return replace(base, search=replace(base.search, **kwargs))
+
+
+# ---------------------------------------------------------------- 结果落盘
+def write_json(path: Path, payload) -> Path:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, ensure_ascii=False, indent=2,
+                               allow_nan=False) + "\n", encoding="utf-8")
+    return path
+
+
+def write_text(path: Path, text: str) -> Path:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(text, encoding="utf-8")
+    return path
+
+
+# ---------------------------------------------------------------- 指标工具
+def mean(values) -> float:
+    values = list(values)
+    return round(statistics.fmean(values), 4) if values else 0.0
+
+
+def median(values) -> float:
+    values = list(values)
+    return round(statistics.median(values), 4) if values else 0.0
+
+
+def ratio(hit: int, total: int) -> float:
+    return round(hit / total, 4) if total else 0.0
+
+
+def relative_change(value: float, reference: float) -> float | None:
+    if reference == 0:
+        return None
+    return round(value / reference - 1.0, 4)
+
+
+def markdown_table(headers: list[str], rows: list[list[str]]) -> str:
+    lines = ["| " + " | ".join(headers) + " |",
+             "| " + " | ".join("---" for _ in headers) + " |"]
+    for row in rows:
+        lines.append("| " + " | ".join(str(cell) for cell in row) + " |")
+    return "\n".join(lines)
+
+
+def _num(value, digits: int = 2) -> str:
+    if value is None:
+        return "—"
+    if isinstance(value, bool):
+        return "✔" if value else "✘"
+    if isinstance(value, float):
+        return f"{value:.{digits}f}"
+    return str(value)
+
+
+# ---------------------------------------------------------------- 第三/四问闭环
+def run_loop_episode(loop_cls, scene_spec, config: ProblemConfig) -> dict:
+    """在冻结场景上跑一局闭环，返回逐局指标行（与 ``scripts/run_suite.py`` 同口径）。"""
+    sources = scene_spec.sources(config)          # 真值：仅测试端可见
+    simulator = LocalSimulator(sources, robot_id="COMPARE", config=config,
+                               error_seed=scene_spec.error_seed)
+    client = SimulatorClient("COMPARE", config=config,
+                             transport=LocalSimTransport(simulator))
+    start = perf_counter()
+    loop = loop_cls(client, config)
+    summary = loop.run()
+    compute_wall_s = perf_counter() - start
+    report = recompute_virtual_time(client.log.action_events())
+
+    truth = sorted(source.channel for source in sources)
+    cleared = sorted(summary["cleared_channels"])
+    absent = sorted(summary["absent_channels"])
+    misjudged = sorted(set(absent) & set(truth))
+    valid = (cleared == truth and not misjudged
+             and summary["exit_reason"] == "user_exit"
+             and not summary["anomalies"]
+             and not report["mismatches"])
+    return {
+        "scene_id": scene_spec.scene_id,
+        "problem": scene_spec.problem,
+        "kind": scene_spec.kind,
+        "sources": len(truth),
+        "directional": scene_spec.directional,
+        "cleared": len(cleared),
+        "cleared_ratio": ratio(len(cleared), len(truth)),
+        "cleared_matches_truth": cleared == truth,
+        "absent_misjudged": len(misjudged),
+        "virtual_time_s": round(summary["virtual_time_s"], 3),
+        "accounted_virtual_time_s": (
+            round(summary["accounted_virtual_time_s"], 3)
+            if summary.get("accounted_virtual_time_s") is not None else None),
+        "measures": summary["measures"],
+        "clears": summary["clears"],
+        "clear_success": summary["clear_success"],
+        "total_requests": summary["total_requests"],
+        "move_distance_m": round(summary["move_distance_m"], 2),
+        "fallback_clear_attempts": summary.get("fallback_clear_attempts", 0),
+        "coverage_points": summary.get("coverage_points", 0),
+        "stop_reason": summary["stop_reason"],
+        "exit_reason": summary["exit_reason"],
+        "anomaly_count": len(summary["anomalies"]),
+        "virtual_recompute_mismatches": len(report["mismatches"]),
+        "compute_wall_s": round(compute_wall_s, 3),
+        "valid": valid,
+    }
+
+
+# 第三/四问统一汇报列（顺序即 Markdown 列序）
+EPISODE_COLUMNS = (
+    "cleared_ratio", "cleared_matches_truth", "absent_misjudged",
+    "virtual_time_s", "measures", "total_requests", "move_distance_m",
+    "fallback_clear_attempts", "compute_wall_s", "valid",
+)
+
+
+def aggregate_episodes(rows: list[dict], labels: list[str],
+                       base_label: str | None = None) -> dict:
+    """逐方法汇总 + 相对基线的配对变化（按 scene_id 对齐）。"""
+    summary: dict[str, dict] = {}
+    for label in labels:
+        part = [row for row in rows if row["variant"] == label]
+        if not part:
+            continue
+        summary[label] = {
+            "episodes": len(part),
+            "valid_episodes": sum(1 for row in part if row["valid"]),
+            "cleared_ratio_mean": mean(row["cleared_ratio"] for row in part),
+            "cleared_matches_truth": sum(1 for row in part
+                                         if row["cleared_matches_truth"]),
+            "absent_misjudged_total": sum(row["absent_misjudged"] for row in part),
+            "virtual_time_mean_s": mean(row["virtual_time_s"] for row in part),
+            "virtual_time_median_s": median(row["virtual_time_s"] for row in part),
+            "virtual_time_max_s": round(max(row["virtual_time_s"] for row in part), 3),
+            "measures_mean": mean(row["measures"] for row in part),
+            "total_requests_mean": mean(row["total_requests"] for row in part),
+            "move_distance_mean_m": mean(row["move_distance_m"] for row in part),
+            "fallback_clear_attempts_mean": mean(
+                row["fallback_clear_attempts"] for row in part),
+            "compute_wall_mean_s": mean(row["compute_wall_s"] for row in part),
+        }
+    paired: dict[str, dict] = {}
+    if base_label and len(labels) > 1:
+        base_rows = {row["scene_id"]: row for row in rows
+                     if row["variant"] == base_label}
+        for label in labels:
+            if label == base_label:
+                continue
+            deltas, faster, valid_pairs = [], 0, 0
+            for row in rows:
+                if row["variant"] != label or row["scene_id"] not in base_rows:
+                    continue
+                base_row = base_rows[row["scene_id"]]
+                reference = base_row["virtual_time_s"]
+                deltas.append(relative_change(row["virtual_time_s"], reference))
+                if not (row["valid"] and base_row["valid"]):
+                    continue
+                valid_pairs += 1
+                if row["virtual_time_s"] < reference:
+                    faster += 1
+            paired[f"{label}_vs_{base_label}"] = {
+                "mean_relative_change": mean(d for d in deltas if d is not None),
+                "median_relative_change": median(d for d in deltas if d is not None),
+                "faster_scenes": faster,
+                "compared": len(deltas),
+                "both_valid_scenes": valid_pairs,
+            }
+    return {"variants": summary, "paired": paired}
+
+
+def episodes_markdown(title: str, rows: list[dict], labels: list[str],
+                      base_label: str | None = None) -> str:
+    """逐场景明细 + 汇总表。"""
+    headers = ["scene", "sources", "variant", "cleared", "T(s)", "检测",
+               "请求", "位移(m)", "后备清除", "墙钟(s)", "有效"]
+    body = []
+    for row in sorted(rows, key=lambda r: (r["scene_id"], labels.index(r["variant"]))):
+        body.append([
+            row["scene_id"], row["sources"], row["variant"],
+            f"{row['cleared']}/{row['sources']}",
+            _num(row["virtual_time_s"], 1), row["measures"],
+            row["total_requests"], _num(row["move_distance_m"], 1),
+            row["fallback_clear_attempts"], _num(row["compute_wall_s"], 2),
+            _num(row["valid"]),
+        ])
+    text = [f"# {title}", "", "## 逐场景明细", "",
+            markdown_table(headers, body), "", "## 逐方法汇总", ""]
+    agg = aggregate_episodes(rows, labels, base_label)
+    agg_headers = ["方法", "有效局", "全清率", "全清对数", "缺席误判",
+                   "T 均值(s)", "T 中位(s)", "T 最大(s)", "检测均值",
+                   "请求均值", "位移均值(m)", "后备清除均值", "墙钟均值(s)"]
+    agg_rows = []
+    for label in labels:
+        s = agg["variants"].get(label)
+        if not s:
+            continue
+        agg_rows.append([
+            label, f"{s['valid_episodes']}/{s['episodes']}",
+            _num(s["cleared_ratio_mean"]), s["cleared_matches_truth"],
+            s["absent_misjudged_total"], _num(s["virtual_time_mean_s"], 1),
+            _num(s["virtual_time_median_s"], 1), _num(s["virtual_time_max_s"], 1),
+            _num(s["measures_mean"], 1), _num(s["total_requests_mean"], 1),
+            _num(s["move_distance_mean_m"], 1),
+            _num(s["fallback_clear_attempts_mean"], 2),
+            _num(s["compute_wall_mean_s"], 2),
+        ])
+    text.append(markdown_table(agg_headers, agg_rows))
+    if agg["paired"]:
+        text += ["", "## 相对基线的配对变化（虚拟时间）", ""]
+        pair_rows = [[name, f"{v['mean_relative_change']*100:+.1f}%",
+                      f"{v['median_relative_change']*100:+.1f}%",
+                      f"{v['faster_scenes']}/{v['compared']}",
+                      v["both_valid_scenes"]]
+                     for name, v in agg["paired"].items()]
+        text.append(markdown_table(
+            ["配对", "均值变化", "中位变化", "更快场景", "双方均有效"], pair_rows))
+        invalid = [name for name, value in agg["paired"].items()
+                   if value["both_valid_scenes"] < value["compared"]]
+        if invalid:
+            text += ["",
+                     "> 注：" + "、".join(invalid)
+                     + " 存在未通过有效性判定的配对；其虚拟时间仅作参考，"
+                       "基线未完成全清任务时不构成「更快」证据"
+                       "（以「双方均有效」列为准）。"]
+    return "\n".join(text) + "\n"
+
+
+def json_safe(payload):
+    """把 numpy 标量等转成 json 可写类型（防御式，非必需）。"""
+    return json.loads(json.dumps(payload, ensure_ascii=False, default=float))
+
+
+# ---------------------------------------------------------------- 闭环批量执行
+def execute_variants(scenes, variants, config: ProblemConfig) -> list[dict]:
+    """对每个场景跑遍所有变体（串行；对比规模小，进程池收益有限且便于复现）。"""
+    rows = []
+    for scene in scenes:
+        for label, loop_cls in variants:
+            row = run_loop_episode(loop_cls, scene, config)
+            row["variant"] = label
+            rows.append(row)
+    return rows
+
+
+def closed_loop_payload(question: int, suite_name: str, scenes, variants,
+                        rows: list[dict], labels: list[str],
+                        base_label: str, config: ProblemConfig) -> dict:
+    return {
+        "question": question,
+        "suite": suite_name,
+        "scenes": [scene.as_dict() for scene in scenes],
+        "variants": [{"label": label, "class": loop_cls.__name__}
+                     for label, loop_cls in variants],
+        "config": {
+            "persistent_search": config.search.persistent_search,
+            "bearing_rounding_margin_deg": config.search.bearing_rounding_margin_deg,
+            "q4_coverage_plan": config.search.q4_coverage_plan,
+        },
+        "rows": rows,
+        "aggregate": aggregate_episodes(rows, labels, base_label),
+    }