yes fuking shit
This commit is contained in:
Horldsence
@@ -0,0 +1,361 @@
|
||||
"""冻结基准集的配对 runner：并行执行、逐局留档、有效性判定与配对统计。
|
||||
|
||||
取代散落的 benchmark_*.py 口径分叉（docs/第三四问架构优化与运行提速方案.md §P1-3）：
|
||||
|
||||
- 场景来自 :mod:`mcm26b.harness.suites`（版本化清单，真值只在测试端）；
|
||||
- 每个 (场景, 变体) 一局，落 manifest / results / aggregate(.json/.csv)；
|
||||
- 有效性定义与既有脚本一致且不得放宽：
|
||||
``全清 ∧ 缺席无误判 ∧ 正常退出 ∧ 零异常 ∧ 逐微秒计时复算一致``；
|
||||
- 任何一局无效即 ``all_valid=false`` 且进程非零退出（失败不静默丢样本）；
|
||||
- 变体支持 ``名称@计划名`` 语法，用于同场配对比较两个覆盖计划；
|
||||
- ``--jobs`` 走进程池并行（每进程独立内存回环，不共享端口）。
|
||||
|
||||
uv run python scripts/run_suite.py --problem 3 --suite core --jobs 6
|
||||
uv run python scripts/run_suite.py --problem 4 --suite all --jobs 6 \
|
||||
--variants directional@square49,directional@q4_mincover_v1 \
|
||||
--label q4-plan-ab
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import statistics
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from time import perf_counter
|
||||
|
||||
from mcm26b.config import ProblemConfig
|
||||
from mcm26b.evaluation import recompute_virtual_time
|
||||
from mcm26b.harness import LocalSimTransport, LocalSimulator
|
||||
from mcm26b.harness.suites import (SUITE_VERSION, SceneSpec, suite,
|
||||
suite_manifest)
|
||||
from mcm26b.planner import DirectionalExtension
|
||||
from mcm26b.planner.bayes_loop import BayesLoop
|
||||
from mcm26b.planner.seven import SevenBatch
|
||||
from mcm26b.protocol.client import SimulatorClient
|
||||
from mcm26b.routing import route_metrics
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
Q3_STRATEGIES = {"bayes": BayesLoop, "seven_batch": SevenBatch,
|
||||
"seven_immediate": SevenBatch}
|
||||
Q4_STRATEGIES = {"directional": DirectionalExtension}
|
||||
UNDER_TARGET_S = 4000.0
|
||||
|
||||
|
||||
def build_config(margin_deg: float, q4_plan: str | None,
|
||||
bayes_overrides: dict | None = None,
|
||||
search_overrides: dict | None = None) -> ProblemConfig:
|
||||
"""基准配置：persistent 关闭（对齐"首次证明找全"口径）。"""
|
||||
base = ProblemConfig()
|
||||
search = replace(base.search, bearing_rounding_margin_deg=margin_deg,
|
||||
persistent_search=False)
|
||||
if q4_plan:
|
||||
search = replace(search, q4_coverage_plan=q4_plan)
|
||||
if search_overrides:
|
||||
search = replace(search, **search_overrides)
|
||||
bayes = base.bayes
|
||||
if bayes_overrides:
|
||||
bayes = replace(bayes, **bayes_overrides)
|
||||
return replace(base, search=search, bayes=bayes)
|
||||
|
||||
|
||||
def _coerce(text: str):
|
||||
"""变体覆盖值的字面量解析（true/false/整数/浮点/元组/字符串）。"""
|
||||
lowered = text.lower()
|
||||
if lowered in ("true", "yes", "on"):
|
||||
return True
|
||||
if lowered in ("false", "no", "off"):
|
||||
return False
|
||||
if text.startswith("(") and text.endswith(")"):
|
||||
inner = text[1:-1].strip()
|
||||
return () if not inner else tuple(
|
||||
_coerce(chunk.strip()) for chunk in inner.split(","))
|
||||
for cast in (int, float):
|
||||
try:
|
||||
return cast(text)
|
||||
except ValueError:
|
||||
continue
|
||||
return text
|
||||
|
||||
|
||||
def parse_variant(spec: str, problem: int) -> tuple[str, str, str | None, dict, dict]:
|
||||
"""``名称[@计划][#键=值;键=值]`` → (标签, 策略名, 计划, bayes 覆盖, search 覆盖)。
|
||||
|
||||
变体覆盖用于同场配对比较单个开关，例如
|
||||
``bayes#fuse_localize_into_tour=false``；``search.`` 前缀写入 SearchConfig。
|
||||
"""
|
||||
body, _, override_text = spec.partition("#")
|
||||
name, _, plan = body.partition("@")
|
||||
strategies = Q3_STRATEGIES if problem == 3 else Q4_STRATEGIES
|
||||
if name not in strategies:
|
||||
raise ValueError(f"未知策略 {name!r}（可选 {sorted(strategies)}）")
|
||||
bayes_overrides: dict = {}
|
||||
search_overrides: dict = {}
|
||||
for item in filter(None, (chunk.strip() for chunk in override_text.split(";"))):
|
||||
key, _, value = item.partition("=")
|
||||
if not value:
|
||||
raise ValueError(f"变体覆盖须为 键=值：{item!r}")
|
||||
if key.startswith("search."):
|
||||
search_overrides[key[len("search."):]] = _coerce(value)
|
||||
else:
|
||||
bayes_overrides[key] = _coerce(value)
|
||||
label = spec.replace(" ", "")
|
||||
return label, name, plan or None, bayes_overrides, search_overrides
|
||||
|
||||
|
||||
def run_episode(payload: dict) -> dict:
|
||||
"""执行一局并返回结果行（进程池 worker，必须可 pickle）。"""
|
||||
spec = SceneSpec(**payload["scene"])
|
||||
label = payload["label"]
|
||||
problem = spec.problem
|
||||
config = build_config(payload["margin_deg"], payload["plan"],
|
||||
payload.get("bayes"), payload.get("search"))
|
||||
strategies = Q3_STRATEGIES if problem == 3 else Q4_STRATEGIES
|
||||
loop_cls = strategies[payload["strategy"]]
|
||||
sources = spec.sources(config)
|
||||
simulator = LocalSimulator(sources, robot_id="SUITE", config=config,
|
||||
error_seed=spec.error_seed)
|
||||
client = SimulatorClient("SUITE", config=config,
|
||||
transport=LocalSimTransport(simulator))
|
||||
setup_start = perf_counter()
|
||||
loop = loop_cls(client, config)
|
||||
setup_wall_s = perf_counter() - setup_start
|
||||
start = perf_counter()
|
||||
summary = loop.run()
|
||||
compute_wall_s = perf_counter() - start
|
||||
report = recompute_virtual_time(client.log.action_events())
|
||||
truth = sorted(source.channel for source in sources)
|
||||
cleared = sorted(summary["cleared_channels"])
|
||||
absent = sorted(summary["absent_channels"])
|
||||
valid = (cleared == truth
|
||||
and not (set(absent) & set(truth))
|
||||
and summary["exit_reason"] == "user_exit"
|
||||
and not summary["anomalies"]
|
||||
and not report["mismatches"])
|
||||
routing = _routing_metrics(summary, config)
|
||||
return {
|
||||
"scene_id": spec.scene_id, "variant": label, "problem": problem,
|
||||
"kind": spec.kind, "sources": spec.count, "directional": spec.directional,
|
||||
"virtual_time_s": summary["virtual_time_s"],
|
||||
"accounted_virtual_time_s": summary["accounted_virtual_time_s"],
|
||||
"measures": summary["measures"], "clears": summary["clears"],
|
||||
"clear_success": summary["clear_success"],
|
||||
"total_requests": summary["total_requests"],
|
||||
"move_distance_m": summary["move_distance_m"],
|
||||
"coverage_points": summary["coverage_points"],
|
||||
"fallback_clear_attempts": summary.get("fallback_clear_attempts", 0),
|
||||
"redundant_rescan_measures": summary["redundant_rescan_measures"],
|
||||
"completion_basis": summary["completion_basis"],
|
||||
"stop_reason": summary["stop_reason"], "exit_reason": summary["exit_reason"],
|
||||
"cleared_matches_truth": cleared == truth,
|
||||
"absent_misjudged": sorted(set(absent) & set(truth)),
|
||||
"anomaly_count": len(summary["anomalies"]),
|
||||
"virtual_recompute_mismatches": len(report["mismatches"]),
|
||||
"compute_wall_s": round(compute_wall_s, 3),
|
||||
"plan_verify_wall_s": round(setup_wall_s, 3),
|
||||
"valid": valid,
|
||||
**routing,
|
||||
}
|
||||
|
||||
|
||||
def _routing_metrics(summary: dict, config: ProblemConfig) -> dict:
|
||||
"""S1 必报路由列：作业点去重、离线最优开放回路、MST 下界、缺口与探测静默率。
|
||||
|
||||
作业点 = 发生过 /measure 或 /clear 的位置（去重到 0.5 m）；离线最优是
|
||||
"同一批作业点"上的开放路径最优值，用来把"排线不够好"与"点选得不对"分开。
|
||||
"""
|
||||
rows = summary.get("steps") or []
|
||||
positions = []
|
||||
for row in rows:
|
||||
position = row.get("position")
|
||||
if isinstance(position, dict):
|
||||
positions.append((float(position["x"]), float(position["y"])))
|
||||
elif position is not None:
|
||||
positions.append((float(position[0]), float(position[1])))
|
||||
speed = config.cost.speed_mps
|
||||
metrics = route_metrics(positions, summary["move_distance_m"], speed)
|
||||
measures = [row for row in rows if row.get("action") == "measure"]
|
||||
silent = sum(1 for row in measures if row.get("result") == "no_signal")
|
||||
metrics["probe_no_signal_ratio"] = (
|
||||
round(silent / len(measures), 4) if measures else 0.0)
|
||||
plan_points = summary.get("coverage_plan_points") or []
|
||||
if plan_points:
|
||||
visited = 0
|
||||
for (px, py) in plan_points:
|
||||
if any(abs(px - x) <= 1.0 and abs(py - y) <= 1.0
|
||||
for x, y in positions):
|
||||
visited += 1
|
||||
metrics["coverage_actual_visited"] = visited
|
||||
else:
|
||||
metrics["coverage_actual_visited"] = 0
|
||||
return metrics
|
||||
|
||||
|
||||
def _percentile(values: list[float], fraction: float) -> float:
|
||||
if not values:
|
||||
return 0.0
|
||||
ordered = sorted(values)
|
||||
index = min(len(ordered) - 1, max(0, int(round(fraction * (len(ordered) - 1)))))
|
||||
return ordered[index]
|
||||
|
||||
|
||||
def aggregate(rows: list[dict], labels: list[str]) -> dict:
|
||||
"""逐变体汇总 + 相对首变体的配对统计。"""
|
||||
summary: dict[str, dict] = {}
|
||||
for label in labels:
|
||||
part = [row for row in rows if row["variant"] == label]
|
||||
if not part:
|
||||
continue
|
||||
times = [row["virtual_time_s"] for row in part]
|
||||
summary[label] = {
|
||||
"episodes": len(part),
|
||||
"valid_episodes": sum(1 for row in part if row["valid"]),
|
||||
"virtual_time_mean_s": round(statistics.fmean(times), 1),
|
||||
"virtual_time_median_s": round(statistics.median(times), 1),
|
||||
"virtual_time_max_s": round(max(times), 1),
|
||||
"virtual_time_p90_s": round(_percentile(times, 0.9), 1),
|
||||
"under_4000_ratio": round(
|
||||
sum(1 for t in times if t < UNDER_TARGET_S) / len(times), 3),
|
||||
"measures_mean": round(statistics.fmean(
|
||||
[row["measures"] for row in part]), 1),
|
||||
"total_requests_mean": round(statistics.fmean(
|
||||
[row["total_requests"] for row in part]), 1),
|
||||
"move_distance_mean_m": round(statistics.fmean(
|
||||
[row["move_distance_m"] for row in part]), 1),
|
||||
"fallback_attempts_mean": round(statistics.fmean(
|
||||
[row["fallback_clear_attempts"] for row in part]), 1),
|
||||
"compute_wall_mean_s": round(statistics.fmean(
|
||||
[row["compute_wall_s"] for row in part]), 3),
|
||||
"plan_verify_wall_mean_s": round(statistics.fmean(
|
||||
[row["plan_verify_wall_s"] for row in part]), 3),
|
||||
"distinct_service_points_mean": round(statistics.fmean(
|
||||
[row["distinct_service_points"] for row in part]), 1),
|
||||
"route_gap_vs_offline_mean": round(statistics.fmean(
|
||||
[row["route_gap_vs_offline"] for row in part]), 4),
|
||||
"route_lower_bound_mean_s": round(statistics.fmean(
|
||||
[row["route_lower_bound_s"] for row in part]), 1),
|
||||
"coverage_actual_visited_mean": round(statistics.fmean(
|
||||
[row["coverage_actual_visited"] for row in part]), 1),
|
||||
"probe_no_signal_ratio_mean": round(statistics.fmean(
|
||||
[row["probe_no_signal_ratio"] for row in part]), 4),
|
||||
}
|
||||
paired: dict[str, dict] = {}
|
||||
if len(labels) >= 2:
|
||||
base = labels[0]
|
||||
base_rows = {row["scene_id"]: row for row in rows
|
||||
if row["variant"] == base}
|
||||
for label in labels[1:]:
|
||||
deltas, faster = [], 0
|
||||
for row in rows:
|
||||
if row["variant"] != label or row["scene_id"] not in base_rows:
|
||||
continue
|
||||
reference = base_rows[row["scene_id"]]["virtual_time_s"]
|
||||
if reference <= 0:
|
||||
continue
|
||||
deltas.append(row["virtual_time_s"] / reference - 1.0)
|
||||
if row["virtual_time_s"] < reference:
|
||||
faster += 1
|
||||
if deltas:
|
||||
paired[f"{label}_vs_{base}"] = {
|
||||
"mean_relative_change": round(statistics.fmean(deltas), 4),
|
||||
"median_relative_change": round(statistics.median(deltas), 4),
|
||||
"faster_scenes": faster, "compared": len(deltas)}
|
||||
return {"variants": summary, "paired": paired}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("--problem", type=int, choices=(3, 4), default=3)
|
||||
parser.add_argument("--suite", choices=("core", "boundary", "all"), default="core")
|
||||
parser.add_argument("--variants", default=None,
|
||||
help="逗号分隔；支持 名称@覆盖计划，默认取该问题的标准变体")
|
||||
parser.add_argument("--q4-plan", default=None, help="第四问覆盖计划（所有变体默认值）")
|
||||
parser.add_argument("--bearing-margin-deg", type=float, default=0.005)
|
||||
parser.add_argument("--bayes-json", default=None,
|
||||
help="BayesConfig 覆盖参数（JSON 字符串或文件路径）")
|
||||
parser.add_argument("--jobs", type=int, default=1, help="并行进程数")
|
||||
parser.add_argument("--label", default="run", help="输出子目录名")
|
||||
parser.add_argument("--out", type=Path, default=None,
|
||||
help="输出根目录，默认 output/suite/")
|
||||
args = parser.parse_args()
|
||||
|
||||
default_variants = "bayes,seven_batch" if args.problem == 3 else "directional"
|
||||
variants = [parse_variant(item.strip(), args.problem)
|
||||
for item in (args.variants or default_variants).split(",") if item.strip()]
|
||||
labels = [label for label, *_ in variants]
|
||||
|
||||
bayes_overrides = None
|
||||
if args.bayes_json:
|
||||
raw = (Path(args.bayes_json).read_text(encoding="utf-8")
|
||||
if Path(args.bayes_json).exists() else args.bayes_json)
|
||||
bayes_overrides = json.loads(raw)
|
||||
print(f"bayes 覆盖参数：{json.dumps(bayes_overrides, ensure_ascii=False)}")
|
||||
|
||||
scenes = suite(args.suite, args.problem)
|
||||
out_dir = (args.out or (ROOT / "output/suite")) / f"q{args.problem}-{args.label}"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
manifest = suite_manifest(args.suite, args.problem)
|
||||
manifest.update({"suite_version": SUITE_VERSION, "label": args.label,
|
||||
"variants": list(
|
||||
(args.variants or default_variants).split(",")),
|
||||
"jobs": args.jobs,
|
||||
"bearing_margin_deg": args.bearing_margin_deg,
|
||||
"q4_plan": args.q4_plan})
|
||||
(out_dir / "manifest.json").write_text(
|
||||
json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
payloads = [
|
||||
{"scene": scene.as_dict(), "label": label, "strategy": strategy,
|
||||
"plan": plan or (args.q4_plan if args.problem == 4 else None),
|
||||
"bayes": {**(bayes_overrides or {}), **variant_bayes},
|
||||
"search": variant_search,
|
||||
"margin_deg": args.bearing_margin_deg}
|
||||
for scene in scenes
|
||||
for label, strategy, plan, variant_bayes, variant_search in variants]
|
||||
print(f"基准集 {args.suite}/q{args.problem}：{len(scenes)} 场景 × "
|
||||
f"{len(variants)} 变体 = {len(payloads)} 局，jobs={args.jobs}", flush=True)
|
||||
|
||||
start = perf_counter()
|
||||
if args.jobs > 1:
|
||||
with ProcessPoolExecutor(max_workers=args.jobs) as pool:
|
||||
rows = list(pool.map(run_episode, payloads))
|
||||
else:
|
||||
rows = [run_episode(payload) for payload in payloads]
|
||||
wall = perf_counter() - start
|
||||
|
||||
rows.sort(key=lambda row: (row["scene_id"], row["variant"]))
|
||||
for row in rows:
|
||||
print(f" {row['scene_id']:28s} {row['variant']:34s} "
|
||||
f"T={row['virtual_time_s']:9.1f} M={row['measures']:4d} "
|
||||
f"wall={row['compute_wall_s']:6.2f}s "
|
||||
f"{'OK' if row['valid'] else 'INVALID'}", flush=True)
|
||||
|
||||
summary = aggregate(rows, labels)
|
||||
summary["all_valid"] = all(row["valid"] for row in rows)
|
||||
summary["wall_seconds"] = round(wall, 2)
|
||||
summary["episodes"] = len(rows)
|
||||
summary["suite"] = args.suite
|
||||
summary["problem"] = args.problem
|
||||
summary["label"] = args.label
|
||||
payload = {"summary": summary, "rows": rows}
|
||||
(out_dir / "results.json").write_text(
|
||||
json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
(out_dir / "aggregate.json").write_text(
|
||||
json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
with (out_dir / "aggregate.csv").open("w", encoding="utf-8", newline="") as handle:
|
||||
writer = csv.writer(handle)
|
||||
header = ["scene_id", "variant", "problem", "kind", "sources", "directional"]
|
||||
header += [key for key in rows[0] if key not in header]
|
||||
writer.writerow(header)
|
||||
for row in rows:
|
||||
writer.writerow([row.get(key, "") for key in header])
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
print(f"\n结果已写入 {out_dir}")
|
||||
return 0 if summary["all_valid"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user