362 lines
17 KiB
Python
362 lines
17 KiB
Python
"""冻结基准集的配对 runner：并行执行、逐局留档、有效性判定与配对统计。
|
||
|
||
取代散落的 benchmark_*.py 口径分叉（论文“实验结果与分析”一节 §P1-3）：
|
||
|
||
- 场景来自 :mod:`mcm26b.harness.suites`（版本化清单，真值只在测试端）；
|
||
- 每个 (场景, 变体) 一局，落 manifest / results / aggregate(.json/.csv)；
|
||
- 有效性定义与既有脚本一致且不得放宽：
|
||
``全清 ∧ 缺席无误判 ∧ 正常退出 ∧ 零异常 ∧ 逐微秒计时复算一致``；
|
||
- 任何一局无效即 ``all_valid=false`` 且进程非零退出（失败不静默丢样本）；
|
||
- 变体支持 ``名称@计划名`` 语法，用于同场配对比较两个覆盖计划；
|
||
- ``--jobs`` 走进程池并行（每进程独立内存回环，不共享端口）。
|
||
|
||
uv run python scripts/run_suite.py --problem 3 --suite core --jobs 6
|
||
uv run python scripts/run_suite.py --problem 4 --suite all --jobs 6 \
|
||
--variants directional@square49,directional@q4_mincover_v1 \
|
||
--label q4-plan-ab
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import csv
|
||
import json
|
||
import statistics
|
||
from concurrent.futures import ProcessPoolExecutor
|
||
from dataclasses import replace
|
||
from pathlib import Path
|
||
from time import perf_counter
|
||
|
||
from mcm26b.config import ProblemConfig
|
||
from mcm26b.evaluation import recompute_virtual_time
|
||
from mcm26b.harness import LocalSimTransport, LocalSimulator
|
||
from mcm26b.harness.suites import (SUITE_VERSION, SceneSpec, suite,
|
||
suite_manifest)
|
||
from mcm26b.planner import DirectionalExtension
|
||
from mcm26b.planner.bayes_loop import BayesLoop
|
||
from mcm26b.planner.seven import SevenBatch
|
||
from mcm26b.protocol.client import SimulatorClient
|
||
from mcm26b.routing import route_metrics
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
Q3_STRATEGIES = {"bayes": BayesLoop, "seven_batch": SevenBatch,
|
||
"seven_immediate": SevenBatch}
|
||
Q4_STRATEGIES = {"directional": DirectionalExtension}
|
||
UNDER_TARGET_S = 4000.0
|
||
|
||
|
||
def build_config(margin_deg: float, q4_plan: str | None,
|
||
bayes_overrides: dict | None = None,
|
||
search_overrides: dict | None = None) -> ProblemConfig:
|
||
"""基准配置：persistent 关闭（对齐"首次证明找全"口径）。"""
|
||
base = ProblemConfig()
|
||
search = replace(base.search, bearing_rounding_margin_deg=margin_deg,
|
||
persistent_search=False)
|
||
if q4_plan:
|
||
search = replace(search, q4_coverage_plan=q4_plan)
|
||
if search_overrides:
|
||
search = replace(search, **search_overrides)
|
||
bayes = base.bayes
|
||
if bayes_overrides:
|
||
bayes = replace(bayes, **bayes_overrides)
|
||
return replace(base, search=search, bayes=bayes)
|
||
|
||
|
||
def _coerce(text: str):
|
||
"""变体覆盖值的字面量解析（true/false/整数/浮点/元组/字符串）。"""
|
||
lowered = text.lower()
|
||
if lowered in ("true", "yes", "on"):
|
||
return True
|
||
if lowered in ("false", "no", "off"):
|
||
return False
|
||
if text.startswith("(") and text.endswith(")"):
|
||
inner = text[1:-1].strip()
|
||
return () if not inner else tuple(
|
||
_coerce(chunk.strip()) for chunk in inner.split(","))
|
||
for cast in (int, float):
|
||
try:
|
||
return cast(text)
|
||
except ValueError:
|
||
continue
|
||
return text
|
||
|
||
|
||
def parse_variant(spec: str, problem: int) -> tuple[str, str, str | None, dict, dict]:
|
||
"""``名称[@计划][#键=值;键=值]`` → (标签, 策略名, 计划, bayes 覆盖, search 覆盖)。
|
||
|
||
变体覆盖用于同场配对比较单个开关，例如
|
||
``bayes#fuse_localize_into_tour=false``；``search.`` 前缀写入 SearchConfig。
|
||
"""
|
||
body, _, override_text = spec.partition("#")
|
||
name, _, plan = body.partition("@")
|
||
strategies = Q3_STRATEGIES if problem == 3 else Q4_STRATEGIES
|
||
if name not in strategies:
|
||
raise ValueError(f"未知策略 {name!r}（可选 {sorted(strategies)}）")
|
||
bayes_overrides: dict = {}
|
||
search_overrides: dict = {}
|
||
for item in filter(None, (chunk.strip() for chunk in override_text.split(";"))):
|
||
key, _, value = item.partition("=")
|
||
if not value:
|
||
raise ValueError(f"变体覆盖须为 键=值：{item!r}")
|
||
if key.startswith("search."):
|
||
search_overrides[key[len("search."):]] = _coerce(value)
|
||
else:
|
||
bayes_overrides[key] = _coerce(value)
|
||
label = spec.replace(" ", "")
|
||
return label, name, plan or None, bayes_overrides, search_overrides
|
||
|
||
|
||
def run_episode(payload: dict) -> dict:
|
||
"""执行一局并返回结果行（进程池 worker，必须可 pickle）。"""
|
||
spec = SceneSpec(**payload["scene"])
|
||
label = payload["label"]
|
||
problem = spec.problem
|
||
config = build_config(payload["margin_deg"], payload["plan"],
|
||
payload.get("bayes"), payload.get("search"))
|
||
strategies = Q3_STRATEGIES if problem == 3 else Q4_STRATEGIES
|
||
loop_cls = strategies[payload["strategy"]]
|
||
sources = spec.sources(config)
|
||
simulator = LocalSimulator(sources, robot_id="SUITE", config=config,
|
||
error_seed=spec.error_seed)
|
||
client = SimulatorClient("SUITE", config=config,
|
||
transport=LocalSimTransport(simulator))
|
||
setup_start = perf_counter()
|
||
loop = loop_cls(client, config)
|
||
setup_wall_s = perf_counter() - setup_start
|
||
start = perf_counter()
|
||
summary = loop.run()
|
||
compute_wall_s = perf_counter() - start
|
||
report = recompute_virtual_time(client.log.action_events())
|
||
truth = sorted(source.channel for source in sources)
|
||
cleared = sorted(summary["cleared_channels"])
|
||
absent = sorted(summary["absent_channels"])
|
||
valid = (cleared == truth
|
||
and not (set(absent) & set(truth))
|
||
and summary["exit_reason"] == "user_exit"
|
||
and not summary["anomalies"]
|
||
and not report["mismatches"])
|
||
routing = _routing_metrics(summary, config)
|
||
return {
|
||
"scene_id": spec.scene_id, "variant": label, "problem": problem,
|
||
"kind": spec.kind, "sources": spec.count, "directional": spec.directional,
|
||
"virtual_time_s": summary["virtual_time_s"],
|
||
"accounted_virtual_time_s": summary["accounted_virtual_time_s"],
|
||
"measures": summary["measures"], "clears": summary["clears"],
|
||
"clear_success": summary["clear_success"],
|
||
"total_requests": summary["total_requests"],
|
||
"move_distance_m": summary["move_distance_m"],
|
||
"coverage_points": summary["coverage_points"],
|
||
"fallback_clear_attempts": summary.get("fallback_clear_attempts", 0),
|
||
"redundant_rescan_measures": summary["redundant_rescan_measures"],
|
||
"completion_basis": summary["completion_basis"],
|
||
"stop_reason": summary["stop_reason"], "exit_reason": summary["exit_reason"],
|
||
"cleared_matches_truth": cleared == truth,
|
||
"absent_misjudged": sorted(set(absent) & set(truth)),
|
||
"anomaly_count": len(summary["anomalies"]),
|
||
"virtual_recompute_mismatches": len(report["mismatches"]),
|
||
"compute_wall_s": round(compute_wall_s, 3),
|
||
"plan_verify_wall_s": round(setup_wall_s, 3),
|
||
"valid": valid,
|
||
**routing,
|
||
}
|
||
|
||
|
||
def _routing_metrics(summary: dict, config: ProblemConfig) -> dict:
|
||
"""S1 必报路由列：作业点去重、离线最优开放回路、MST 下界、缺口与探测静默率。
|
||
|
||
作业点 = 发生过 /measure 或 /clear 的位置（去重到 0.5 m）；离线最优是
|
||
"同一批作业点"上的开放路径最优值，用来把"排线不够好"与"点选得不对"分开。
|
||
"""
|
||
rows = summary.get("steps") or []
|
||
positions = []
|
||
for row in rows:
|
||
position = row.get("position")
|
||
if isinstance(position, dict):
|
||
positions.append((float(position["x"]), float(position["y"])))
|
||
elif position is not None:
|
||
positions.append((float(position[0]), float(position[1])))
|
||
speed = config.cost.speed_mps
|
||
metrics = route_metrics(positions, summary["move_distance_m"], speed)
|
||
measures = [row for row in rows if row.get("action") == "measure"]
|
||
silent = sum(1 for row in measures if row.get("result") == "no_signal")
|
||
metrics["probe_no_signal_ratio"] = (
|
||
round(silent / len(measures), 4) if measures else 0.0)
|
||
plan_points = summary.get("coverage_plan_points") or []
|
||
if plan_points:
|
||
visited = 0
|
||
for (px, py) in plan_points:
|
||
if any(abs(px - x) <= 1.0 and abs(py - y) <= 1.0
|
||
for x, y in positions):
|
||
visited += 1
|
||
metrics["coverage_actual_visited"] = visited
|
||
else:
|
||
metrics["coverage_actual_visited"] = 0
|
||
return metrics
|
||
|
||
|
||
def _percentile(values: list[float], fraction: float) -> float:
|
||
if not values:
|
||
return 0.0
|
||
ordered = sorted(values)
|
||
index = min(len(ordered) - 1, max(0, int(round(fraction * (len(ordered) - 1)))))
|
||
return ordered[index]
|
||
|
||
|
||
def aggregate(rows: list[dict], labels: list[str]) -> dict:
|
||
"""逐变体汇总 + 相对首变体的配对统计。"""
|
||
summary: dict[str, dict] = {}
|
||
for label in labels:
|
||
part = [row for row in rows if row["variant"] == label]
|
||
if not part:
|
||
continue
|
||
times = [row["virtual_time_s"] for row in part]
|
||
summary[label] = {
|
||
"episodes": len(part),
|
||
"valid_episodes": sum(1 for row in part if row["valid"]),
|
||
"virtual_time_mean_s": round(statistics.fmean(times), 1),
|
||
"virtual_time_median_s": round(statistics.median(times), 1),
|
||
"virtual_time_max_s": round(max(times), 1),
|
||
"virtual_time_p90_s": round(_percentile(times, 0.9), 1),
|
||
"under_4000_ratio": round(
|
||
sum(1 for t in times if t < UNDER_TARGET_S) / len(times), 3),
|
||
"measures_mean": round(statistics.fmean(
|
||
[row["measures"] for row in part]), 1),
|
||
"total_requests_mean": round(statistics.fmean(
|
||
[row["total_requests"] for row in part]), 1),
|
||
"move_distance_mean_m": round(statistics.fmean(
|
||
[row["move_distance_m"] for row in part]), 1),
|
||
"fallback_attempts_mean": round(statistics.fmean(
|
||
[row["fallback_clear_attempts"] for row in part]), 1),
|
||
"compute_wall_mean_s": round(statistics.fmean(
|
||
[row["compute_wall_s"] for row in part]), 3),
|
||
"plan_verify_wall_mean_s": round(statistics.fmean(
|
||
[row["plan_verify_wall_s"] for row in part]), 3),
|
||
"distinct_service_points_mean": round(statistics.fmean(
|
||
[row["distinct_service_points"] for row in part]), 1),
|
||
"route_gap_vs_offline_mean": round(statistics.fmean(
|
||
[row["route_gap_vs_offline"] for row in part]), 4),
|
||
"route_lower_bound_mean_s": round(statistics.fmean(
|
||
[row["route_lower_bound_s"] for row in part]), 1),
|
||
"coverage_actual_visited_mean": round(statistics.fmean(
|
||
[row["coverage_actual_visited"] for row in part]), 1),
|
||
"probe_no_signal_ratio_mean": round(statistics.fmean(
|
||
[row["probe_no_signal_ratio"] for row in part]), 4),
|
||
}
|
||
paired: dict[str, dict] = {}
|
||
if len(labels) >= 2:
|
||
base = labels[0]
|
||
base_rows = {row["scene_id"]: row for row in rows
|
||
if row["variant"] == base}
|
||
for label in labels[1:]:
|
||
deltas, faster = [], 0
|
||
for row in rows:
|
||
if row["variant"] != label or row["scene_id"] not in base_rows:
|
||
continue
|
||
reference = base_rows[row["scene_id"]]["virtual_time_s"]
|
||
if reference <= 0:
|
||
continue
|
||
deltas.append(row["virtual_time_s"] / reference - 1.0)
|
||
if row["virtual_time_s"] < reference:
|
||
faster += 1
|
||
if deltas:
|
||
paired[f"{label}_vs_{base}"] = {
|
||
"mean_relative_change": round(statistics.fmean(deltas), 4),
|
||
"median_relative_change": round(statistics.median(deltas), 4),
|
||
"faster_scenes": faster, "compared": len(deltas)}
|
||
return {"variants": summary, "paired": paired}
|
||
|
||
|
||
def main() -> int:
|
||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||
parser.add_argument("--problem", type=int, choices=(3, 4), default=3)
|
||
parser.add_argument("--suite", choices=("core", "boundary", "all"), default="core")
|
||
parser.add_argument("--variants", default=None,
|
||
help="逗号分隔；支持 名称@覆盖计划，默认取该问题的标准变体")
|
||
parser.add_argument("--q4-plan", default=None, help="第四问覆盖计划（所有变体默认值）")
|
||
parser.add_argument("--bearing-margin-deg", type=float, default=0.005)
|
||
parser.add_argument("--bayes-json", default=None,
|
||
help="BayesConfig 覆盖参数（JSON 字符串或文件路径）")
|
||
parser.add_argument("--jobs", type=int, default=1, help="并行进程数")
|
||
parser.add_argument("--label", default="run", help="输出子目录名")
|
||
parser.add_argument("--out", type=Path, default=None,
|
||
help="输出根目录，默认 output/suite/")
|
||
args = parser.parse_args()
|
||
|
||
default_variants = "bayes,seven_batch" if args.problem == 3 else "directional"
|
||
variants = [parse_variant(item.strip(), args.problem)
|
||
for item in (args.variants or default_variants).split(",") if item.strip()]
|
||
labels = [label for label, *_ in variants]
|
||
|
||
bayes_overrides = None
|
||
if args.bayes_json:
|
||
raw = (Path(args.bayes_json).read_text(encoding="utf-8")
|
||
if Path(args.bayes_json).exists() else args.bayes_json)
|
||
bayes_overrides = json.loads(raw)
|
||
print(f"bayes 覆盖参数：{json.dumps(bayes_overrides, ensure_ascii=False)}")
|
||
|
||
scenes = suite(args.suite, args.problem)
|
||
out_dir = (args.out or (ROOT / "output/suite")) / f"q{args.problem}-{args.label}"
|
||
out_dir.mkdir(parents=True, exist_ok=True)
|
||
manifest = suite_manifest(args.suite, args.problem)
|
||
manifest.update({"suite_version": SUITE_VERSION, "label": args.label,
|
||
"variants": list(
|
||
(args.variants or default_variants).split(",")),
|
||
"jobs": args.jobs,
|
||
"bearing_margin_deg": args.bearing_margin_deg,
|
||
"q4_plan": args.q4_plan})
|
||
(out_dir / "manifest.json").write_text(
|
||
json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
|
||
payloads = [
|
||
{"scene": scene.as_dict(), "label": label, "strategy": strategy,
|
||
"plan": plan or (args.q4_plan if args.problem == 4 else None),
|
||
"bayes": {**(bayes_overrides or {}), **variant_bayes},
|
||
"search": variant_search,
|
||
"margin_deg": args.bearing_margin_deg}
|
||
for scene in scenes
|
||
for label, strategy, plan, variant_bayes, variant_search in variants]
|
||
print(f"基准集 {args.suite}/q{args.problem}：{len(scenes)} 场景 × "
|
||
f"{len(variants)} 变体 = {len(payloads)} 局，jobs={args.jobs}", flush=True)
|
||
|
||
start = perf_counter()
|
||
if args.jobs > 1:
|
||
with ProcessPoolExecutor(max_workers=args.jobs) as pool:
|
||
rows = list(pool.map(run_episode, payloads))
|
||
else:
|
||
rows = [run_episode(payload) for payload in payloads]
|
||
wall = perf_counter() - start
|
||
|
||
rows.sort(key=lambda row: (row["scene_id"], row["variant"]))
|
||
for row in rows:
|
||
print(f" {row['scene_id']:28s} {row['variant']:34s} "
|
||
f"T={row['virtual_time_s']:9.1f} M={row['measures']:4d} "
|
||
f"wall={row['compute_wall_s']:6.2f}s "
|
||
f"{'OK' if row['valid'] else 'INVALID'}", flush=True)
|
||
|
||
summary = aggregate(rows, labels)
|
||
summary["all_valid"] = all(row["valid"] for row in rows)
|
||
summary["wall_seconds"] = round(wall, 2)
|
||
summary["episodes"] = len(rows)
|
||
summary["suite"] = args.suite
|
||
summary["problem"] = args.problem
|
||
summary["label"] = args.label
|
||
payload = {"summary": summary, "rows": rows}
|
||
(out_dir / "results.json").write_text(
|
||
json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
(out_dir / "aggregate.json").write_text(
|
||
json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
with (out_dir / "aggregate.csv").open("w", encoding="utf-8", newline="") as handle:
|
||
writer = csv.writer(handle)
|
||
header = ["scene_id", "variant", "problem", "kind", "sources", "directional"]
|
||
header += [key for key in rows[0] if key not in header]
|
||
writer.writerow(header)
|
||
for row in rows:
|
||
writer.writerow([row.get(key, "") for key in header])
|
||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||
print(f"\n结果已写入 {out_dir}")
|
||
return 0 if summary["all_valid"] else 1
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|