"""冻结基准集的配对 runner：并行执行、逐局留档、有效性判定与配对统计。 取代散落的 benchmark_*.py 口径分叉（docs/第三四问架构优化与运行提速方案.md §P1-3）： - 场景来自 :mod:`mcm26b.harness.suites`（版本化清单，真值只在测试端）； - 每个 (场景, 变体) 一局，落 manifest / results / aggregate(.json/.csv)； - 有效性定义与既有脚本一致且不得放宽： ``全清 ∧ 缺席无误判 ∧ 正常退出 ∧ 零异常 ∧ 逐微秒计时复算一致``； - 任何一局无效即 ``all_valid=false`` 且进程非零退出（失败不静默丢样本）； - 变体支持 ``名称@计划名`` 语法，用于同场配对比较两个覆盖计划； - ``--jobs`` 走进程池并行（每进程独立内存回环，不共享端口）。 uv run python scripts/run_suite.py --problem 3 --suite core --jobs 6 uv run python scripts/run_suite.py --problem 4 --suite all --jobs 6 \ --variants directional@square49,directional@q4_mincover_v1 \ --label q4-plan-ab """ from __future__ import annotations import argparse import csv import json import statistics from concurrent.futures import ProcessPoolExecutor from dataclasses import replace from pathlib import Path from time import perf_counter from mcm26b.config import ProblemConfig from mcm26b.evaluation import recompute_virtual_time from mcm26b.harness import LocalSimTransport, LocalSimulator from mcm26b.harness.suites import (SUITE_VERSION, SceneSpec, suite, suite_manifest) from mcm26b.planner import DirectionalExtension from mcm26b.planner.bayes_loop import BayesLoop from mcm26b.planner.seven import SevenBatch from mcm26b.protocol.client import SimulatorClient from mcm26b.routing import route_metrics ROOT = Path(__file__).resolve().parents[1] Q3_STRATEGIES = {"bayes": BayesLoop, "seven_batch": SevenBatch, "seven_immediate": SevenBatch} Q4_STRATEGIES = {"directional": DirectionalExtension} UNDER_TARGET_S = 4000.0 def build_config(margin_deg: float, q4_plan: str | None, bayes_overrides: dict | None = None, search_overrides: dict | None = None) -> ProblemConfig: """基准配置：persistent 关闭（对齐"首次证明找全"口径）。""" base = ProblemConfig() search = replace(base.search, bearing_rounding_margin_deg=margin_deg, persistent_search=False) if q4_plan: search = replace(search, q4_coverage_plan=q4_plan) if search_overrides: search = replace(search, **search_overrides) bayes = base.bayes if bayes_overrides: bayes = replace(bayes, **bayes_overrides) return replace(base, search=search, bayes=bayes) def _coerce(text: str): """变体覆盖值的字面量解析（true/false/整数/浮点/元组/字符串）。""" lowered = text.lower() if lowered in ("true", "yes", "on"): return True if lowered in ("false", "no", "off"): return False if text.startswith("(") and text.endswith(")"): inner = text[1:-1].strip() return () if not inner else tuple( _coerce(chunk.strip()) for chunk in inner.split(",")) for cast in (int, float): try: return cast(text) except ValueError: continue return text def parse_variant(spec: str, problem: int) -> tuple[str, str, str | None, dict, dict]: """``名称[@计划][#键=值;键=值]`` → (标签, 策略名, 计划, bayes 覆盖, search 覆盖)。 变体覆盖用于同场配对比较单个开关，例如 ``bayes#fuse_localize_into_tour=false``；``search.`` 前缀写入 SearchConfig。 """ body, _, override_text = spec.partition("#") name, _, plan = body.partition("@") strategies = Q3_STRATEGIES if problem == 3 else Q4_STRATEGIES if name not in strategies: raise ValueError(f"未知策略 {name!r}（可选 {sorted(strategies)}）") bayes_overrides: dict = {} search_overrides: dict = {} for item in filter(None, (chunk.strip() for chunk in override_text.split(";"))): key, _, value = item.partition("=") if not value: raise ValueError(f"变体覆盖须为 键=值：{item!r}") if key.startswith("search."): search_overrides[key[len("search."):]] = _coerce(value) else: bayes_overrides[key] = _coerce(value) label = spec.replace(" ", "") return label, name, plan or None, bayes_overrides, search_overrides def run_episode(payload: dict) -> dict: """执行一局并返回结果行（进程池 worker，必须可 pickle）。""" spec = SceneSpec(**payload["scene"]) label = payload["label"] problem = spec.problem config = build_config(payload["margin_deg"], payload["plan"], payload.get("bayes"), payload.get("search")) strategies = Q3_STRATEGIES if problem == 3 else Q4_STRATEGIES loop_cls = strategies[payload["strategy"]] sources = spec.sources(config) simulator = LocalSimulator(sources, robot_id="SUITE", config=config, error_seed=spec.error_seed) client = SimulatorClient("SUITE", config=config, transport=LocalSimTransport(simulator)) setup_start = perf_counter() loop = loop_cls(client, config) setup_wall_s = perf_counter() - setup_start start = perf_counter() summary = loop.run() compute_wall_s = perf_counter() - start report = recompute_virtual_time(client.log.action_events()) truth = sorted(source.channel for source in sources) cleared = sorted(summary["cleared_channels"]) absent = sorted(summary["absent_channels"]) valid = (cleared == truth and not (set(absent) & set(truth)) and summary["exit_reason"] == "user_exit" and not summary["anomalies"] and not report["mismatches"]) routing = _routing_metrics(summary, config) return { "scene_id": spec.scene_id, "variant": label, "problem": problem, "kind": spec.kind, "sources": spec.count, "directional": spec.directional, "virtual_time_s": summary["virtual_time_s"], "accounted_virtual_time_s": summary["accounted_virtual_time_s"], "measures": summary["measures"], "clears": summary["clears"], "clear_success": summary["clear_success"], "total_requests": summary["total_requests"], "move_distance_m": summary["move_distance_m"], "coverage_points": summary["coverage_points"], "fallback_clear_attempts": summary.get("fallback_clear_attempts", 0), "redundant_rescan_measures": summary["redundant_rescan_measures"], "completion_basis": summary["completion_basis"], "stop_reason": summary["stop_reason"], "exit_reason": summary["exit_reason"], "cleared_matches_truth": cleared == truth, "absent_misjudged": sorted(set(absent) & set(truth)), "anomaly_count": len(summary["anomalies"]), "virtual_recompute_mismatches": len(report["mismatches"]), "compute_wall_s": round(compute_wall_s, 3), "plan_verify_wall_s": round(setup_wall_s, 3), "valid": valid, **routing, } def _routing_metrics(summary: dict, config: ProblemConfig) -> dict: """S1 必报路由列：作业点去重、离线最优开放回路、MST 下界、缺口与探测静默率。 作业点 = 发生过 /measure 或 /clear 的位置（去重到 0.5 m）；离线最优是 "同一批作业点"上的开放路径最优值，用来把"排线不够好"与"点选得不对"分开。 """ rows = summary.get("steps") or [] positions = [] for row in rows: position = row.get("position") if isinstance(position, dict): positions.append((float(position["x"]), float(position["y"]))) elif position is not None: positions.append((float(position[0]), float(position[1]))) speed = config.cost.speed_mps metrics = route_metrics(positions, summary["move_distance_m"], speed) measures = [row for row in rows if row.get("action") == "measure"] silent = sum(1 for row in measures if row.get("result") == "no_signal") metrics["probe_no_signal_ratio"] = ( round(silent / len(measures), 4) if measures else 0.0) plan_points = summary.get("coverage_plan_points") or [] if plan_points: visited = 0 for (px, py) in plan_points: if any(abs(px - x) <= 1.0 and abs(py - y) <= 1.0 for x, y in positions): visited += 1 metrics["coverage_actual_visited"] = visited else: metrics["coverage_actual_visited"] = 0 return metrics def _percentile(values: list[float], fraction: float) -> float: if not values: return 0.0 ordered = sorted(values) index = min(len(ordered) - 1, max(0, int(round(fraction * (len(ordered) - 1))))) return ordered[index] def aggregate(rows: list[dict], labels: list[str]) -> dict: """逐变体汇总 + 相对首变体的配对统计。""" summary: dict[str, dict] = {} for label in labels: part = [row for row in rows if row["variant"] == label] if not part: continue times = [row["virtual_time_s"] for row in part] summary[label] = { "episodes": len(part), "valid_episodes": sum(1 for row in part if row["valid"]), "virtual_time_mean_s": round(statistics.fmean(times), 1), "virtual_time_median_s": round(statistics.median(times), 1), "virtual_time_max_s": round(max(times), 1), "virtual_time_p90_s": round(_percentile(times, 0.9), 1), "under_4000_ratio": round( sum(1 for t in times if t < UNDER_TARGET_S) / len(times), 3), "measures_mean": round(statistics.fmean( [row["measures"] for row in part]), 1), "total_requests_mean": round(statistics.fmean( [row["total_requests"] for row in part]), 1), "move_distance_mean_m": round(statistics.fmean( [row["move_distance_m"] for row in part]), 1), "fallback_attempts_mean": round(statistics.fmean( [row["fallback_clear_attempts"] for row in part]), 1), "compute_wall_mean_s": round(statistics.fmean( [row["compute_wall_s"] for row in part]), 3), "plan_verify_wall_mean_s": round(statistics.fmean( [row["plan_verify_wall_s"] for row in part]), 3), "distinct_service_points_mean": round(statistics.fmean( [row["distinct_service_points"] for row in part]), 1), "route_gap_vs_offline_mean": round(statistics.fmean( [row["route_gap_vs_offline"] for row in part]), 4), "route_lower_bound_mean_s": round(statistics.fmean( [row["route_lower_bound_s"] for row in part]), 1), "coverage_actual_visited_mean": round(statistics.fmean( [row["coverage_actual_visited"] for row in part]), 1), "probe_no_signal_ratio_mean": round(statistics.fmean( [row["probe_no_signal_ratio"] for row in part]), 4), } paired: dict[str, dict] = {} if len(labels) >= 2: base = labels[0] base_rows = {row["scene_id"]: row for row in rows if row["variant"] == base} for label in labels[1:]: deltas, faster = [], 0 for row in rows: if row["variant"] != label or row["scene_id"] not in base_rows: continue reference = base_rows[row["scene_id"]]["virtual_time_s"] if reference <= 0: continue deltas.append(row["virtual_time_s"] / reference - 1.0) if row["virtual_time_s"] < reference: faster += 1 if deltas: paired[f"{label}_vs_{base}"] = { "mean_relative_change": round(statistics.fmean(deltas), 4), "median_relative_change": round(statistics.median(deltas), 4), "faster_scenes": faster, "compared": len(deltas)} return {"variants": summary, "paired": paired} def main() -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--problem", type=int, choices=(3, 4), default=3) parser.add_argument("--suite", choices=("core", "boundary", "all"), default="core") parser.add_argument("--variants", default=None, help="逗号分隔；支持 名称@覆盖计划，默认取该问题的标准变体") parser.add_argument("--q4-plan", default=None, help="第四问覆盖计划（所有变体默认值）") parser.add_argument("--bearing-margin-deg", type=float, default=0.005) parser.add_argument("--bayes-json", default=None, help="BayesConfig 覆盖参数（JSON 字符串或文件路径）") parser.add_argument("--jobs", type=int, default=1, help="并行进程数") parser.add_argument("--label", default="run", help="输出子目录名") parser.add_argument("--out", type=Path, default=None, help="输出根目录，默认 output/suite/") args = parser.parse_args() default_variants = "bayes,seven_batch" if args.problem == 3 else "directional" variants = [parse_variant(item.strip(), args.problem) for item in (args.variants or default_variants).split(",") if item.strip()] labels = [label for label, *_ in variants] bayes_overrides = None if args.bayes_json: raw = (Path(args.bayes_json).read_text(encoding="utf-8") if Path(args.bayes_json).exists() else args.bayes_json) bayes_overrides = json.loads(raw) print(f"bayes 覆盖参数：{json.dumps(bayes_overrides, ensure_ascii=False)}") scenes = suite(args.suite, args.problem) out_dir = (args.out or (ROOT / "output/suite")) / f"q{args.problem}-{args.label}" out_dir.mkdir(parents=True, exist_ok=True) manifest = suite_manifest(args.suite, args.problem) manifest.update({"suite_version": SUITE_VERSION, "label": args.label, "variants": list( (args.variants or default_variants).split(",")), "jobs": args.jobs, "bearing_margin_deg": args.bearing_margin_deg, "q4_plan": args.q4_plan}) (out_dir / "manifest.json").write_text( json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") payloads = [ {"scene": scene.as_dict(), "label": label, "strategy": strategy, "plan": plan or (args.q4_plan if args.problem == 4 else None), "bayes": {**(bayes_overrides or {}), **variant_bayes}, "search": variant_search, "margin_deg": args.bearing_margin_deg} for scene in scenes for label, strategy, plan, variant_bayes, variant_search in variants] print(f"基准集 {args.suite}/q{args.problem}：{len(scenes)} 场景 × " f"{len(variants)} 变体 = {len(payloads)} 局，jobs={args.jobs}", flush=True) start = perf_counter() if args.jobs > 1: with ProcessPoolExecutor(max_workers=args.jobs) as pool: rows = list(pool.map(run_episode, payloads)) else: rows = [run_episode(payload) for payload in payloads] wall = perf_counter() - start rows.sort(key=lambda row: (row["scene_id"], row["variant"])) for row in rows: print(f" {row['scene_id']:28s} {row['variant']:34s} " f"T={row['virtual_time_s']:9.1f} M={row['measures']:4d} " f"wall={row['compute_wall_s']:6.2f}s " f"{'OK' if row['valid'] else 'INVALID'}", flush=True) summary = aggregate(rows, labels) summary["all_valid"] = all(row["valid"] for row in rows) summary["wall_seconds"] = round(wall, 2) summary["episodes"] = len(rows) summary["suite"] = args.suite summary["problem"] = args.problem summary["label"] = args.label payload = {"summary": summary, "rows": rows} (out_dir / "results.json").write_text( json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") (out_dir / "aggregate.json").write_text( json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") with (out_dir / "aggregate.csv").open("w", encoding="utf-8", newline="") as handle: writer = csv.writer(handle) header = ["scene_id", "variant", "problem", "kind", "sources", "directional"] header += [key for key in rows[0] if key not in header] writer.writerow(header) for row in rows: writer.writerow([row.get(key, "") for key in header]) print(json.dumps(summary, ensure_ascii=False, indent=2)) print(f"\n结果已写入 {out_dir}") return 0 if summary["all_valid"] else 1 if __name__ == "__main__": raise SystemExit(main())