159 lines
7.8 KiB
Python
159 lines
7.8 KiB
Python
"""BayesLoop 与 SevenBatch 的配对消融（离线，自建模拟器）。
|
|
|
|
同一场景、同一误差场下配对比较两种策略，输出逐场景行与聚合统计：
|
|
all_cleared / correct_absence / anomalies / 虚拟时间 / 检测数 / 距离 /
|
|
实机计算墙钟。基准配置 persistent_search=False（比较"首次证明找全"的
|
|
虚拟时间口径；persistent 复扫到限时的行为与策略质量无关）。
|
|
|
|
Run: uv run python scripts/benchmark_q3_bayes.py --seeds 0,1,2,3,4,5
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from dataclasses import asdict, replace
|
|
import hashlib
|
|
import json
|
|
import math
|
|
from pathlib import Path
|
|
from statistics import mean, median
|
|
from time import perf_counter
|
|
|
|
from mcm26b.config import ProblemConfig
|
|
from mcm26b.planner.bayes_loop import BayesLoop
|
|
from mcm26b.planner.seven import SevenBatch
|
|
from mcm26b.protocol.client import SimulatorClient
|
|
from mcm26b.sim import LocalSimulator, LocalSimTransport, Source
|
|
from mcm26b.sim.scenes import random_omni_scene
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
VARIANTS = {"seven_batch": SevenBatch, "bayes": BayesLoop}
|
|
|
|
|
|
def run_case(name, sources, error_seed, config, out):
|
|
scene_path = out / "scenes" / f"{name}.json"
|
|
scene_path.parent.mkdir(parents=True, exist_ok=True)
|
|
scene_path.write_text(json.dumps(
|
|
{"sources": [asdict(s) for s in sources], "error_seed": error_seed},
|
|
indent=2), encoding="utf-8")
|
|
rows = []
|
|
for label, cls in VARIANTS.items():
|
|
sim = LocalSimulator(sources, robot_id="Q3-BAYES", config=config,
|
|
error_seed=error_seed)
|
|
client = SimulatorClient("Q3-BAYES", config=config,
|
|
transport=LocalSimTransport(sim))
|
|
loop = cls(client, config)
|
|
start = perf_counter()
|
|
summary = loop.run()
|
|
elapsed = perf_counter() - start
|
|
actions = client.log.action_events()
|
|
log_path = out / "actions" / f"{name}-{label}.jsonl"
|
|
log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
with log_path.open("w", encoding="utf-8") as handle:
|
|
for event in actions:
|
|
handle.write(json.dumps(event, ensure_ascii=False) + "\n")
|
|
truth = {s.channel for s in sources}
|
|
absent = set(range(1, 21)) - truth
|
|
# 缺席证明有效性：PF 缺席以幸存集为空为据，无源频道缺席即为正确
|
|
correct_absence = set(summary["absent_channels"]) <= absent
|
|
accounted = summary.get("accounted_virtual_time_s") or summary["virtual_time_s"]
|
|
row = {"case": name, "variant": label, "source_count": len(sources),
|
|
"all_cleared": sim.cleared == truth, "completed": summary["completed"],
|
|
"correct_absence": correct_absence,
|
|
"normal_exit": summary["exit_reason"] == "user_exit",
|
|
"virtual_time_s": summary["virtual_time_s"],
|
|
"accounted_virtual_time_s": accounted,
|
|
"measures": summary["measures"], "clears": summary["clears"],
|
|
"failed_clears": summary["clears"] - len(sim.cleared),
|
|
"distance_m": summary["move_distance_m"],
|
|
"skipped_measures": summary.get("skipped_measures", 0),
|
|
"no_shrink_measures": summary.get("no_shrink_measures", 0),
|
|
"compute_wall_s": elapsed, "anomalies": summary["anomalies"],
|
|
"stop_reason": summary["stop_reason"],
|
|
"actions_sha256": hashlib.sha256(log_path.read_bytes()).hexdigest()}
|
|
rows.append(row)
|
|
print(f"{name} {label}: cleared={row['all_cleared']} "
|
|
f"T={row['virtual_time_s']:.0f} M={row['measures']} "
|
|
f"D={row['distance_m']:.0f} wall={elapsed:.1f}s", flush=True)
|
|
return rows
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--seeds", default="0,1,2,3,4,5")
|
|
parser.add_argument("--counts", default="10,16")
|
|
parser.add_argument("--out", type=Path, default=ROOT / "output/q3/bayes-ablation")
|
|
parser.add_argument("--bearing-margin-deg", type=float, default=0.005)
|
|
parser.add_argument("--bayes-json", type=Path, default=None,
|
|
help="BayesConfig 覆盖参数 JSON（如 tune_q3_bayes 的"
|
|
" best_params.json）；缺省用库默认配置")
|
|
args = parser.parse_args()
|
|
args.out.mkdir(parents=True, exist_ok=True)
|
|
|
|
base = ProblemConfig()
|
|
bayes_params = {}
|
|
if args.bayes_json is not None:
|
|
bayes_params = json.loads(args.bayes_json.read_text(encoding="utf-8"))
|
|
if "best_params" in bayes_params:
|
|
bayes_params = bayes_params["best_params"]
|
|
base = replace(base,
|
|
bayes=replace(base.bayes, **bayes_params) if bayes_params
|
|
else base.bayes,
|
|
search=replace(base.search,
|
|
bearing_rounding_margin_deg=args.bearing_margin_deg,
|
|
persistent_search=False))
|
|
if bayes_params:
|
|
print("bayes 覆盖参数:", json.dumps(bayes_params, ensure_ascii=False))
|
|
config = base
|
|
rows = []
|
|
seeds = [int(s) for s in args.seeds.split(",")]
|
|
counts = [int(c) for c in args.counts.split(",")]
|
|
for count in counts:
|
|
for seed in seeds:
|
|
rows.extend(run_case(f"random-{count}-seed{seed}",
|
|
random_omni_scene(seed, count, config=config),
|
|
20260910 + seed, config, args.out))
|
|
# 圆域边界源（r_eff=1000，扇区接缝处）：最刁钻的发现与定位场景
|
|
edge = [Source(c + 1, 1800 * math.cos(math.pi / 6 + 2 * math.pi * c / count),
|
|
1800 * math.sin(math.pi / 6 + 2 * math.pi * c / count), 1000)
|
|
for c in range(count)]
|
|
rows.extend(run_case(f"boundary-{count}", edge, 20260910, config, args.out))
|
|
|
|
aggregates = {}
|
|
for label in VARIANTS:
|
|
part = [r for r in rows if r["variant"] == label]
|
|
baseline = {r["case"]: r for r in rows if r["variant"] == "seven_batch"}
|
|
savings = [1 - r["virtual_time_s"] / baseline[r["case"]]["virtual_time_s"]
|
|
for r in part]
|
|
valid = sum(r["all_cleared"] and r["completed"] and r["correct_absence"]
|
|
and r["normal_exit"] and not r["anomalies"] for r in part)
|
|
aggregates[label] = {
|
|
"runs": len(part), "all_cleared_runs": sum(r["all_cleared"] for r in part),
|
|
"valid_runs": valid,
|
|
"mean_virtual_s": mean(r["virtual_time_s"] for r in part),
|
|
"median_virtual_s": median(r["virtual_time_s"] for r in part),
|
|
"max_virtual_s": max(r["virtual_time_s"] for r in part),
|
|
"mean_measures": mean(r["measures"] for r in part),
|
|
"mean_distance_m": mean(r["distance_m"] for r in part),
|
|
"mean_skipped_measures": mean(r["skipped_measures"] for r in part),
|
|
"mean_compute_wall_s": mean(r["compute_wall_s"] for r in part),
|
|
"mean_paired_time_reduction": mean(savings),
|
|
"min_paired_time_reduction": min(savings),
|
|
"max_paired_time_reduction": max(savings),
|
|
"failed_clears": sum(r["failed_clears"] for r in part),
|
|
}
|
|
result = {"source": "local_simulator_only", "seeds": args.seeds,
|
|
"counts": args.counts, "config": asdict(config),
|
|
"aggregates": aggregates, "runs": rows,
|
|
"limitations": "自建模拟器合成数据；persistent 关闭以对齐找全口径。"}
|
|
(args.out / "benchmark.json").write_text(
|
|
json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
print(json.dumps(aggregates, ensure_ascii=False, indent=2))
|
|
ok = all(a["valid_runs"] == a["runs"] for a in aggregates.values()) \
|
|
and aggregates["bayes"]["mean_paired_time_reduction"] > 0
|
|
return 0 if ok else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|