94 lines
4.9 KiB
Python
94 lines
4.9 KiB
Python
"""固定场景配对审核：先保存 baseline，再以相同场景、误差种子比较改进。
|
|
|
|
不连接官方模拟器；真值仅用于本地模拟器和执行后的验收。
|
|
"""
|
|
import argparse
|
|
from dataclasses import asdict
|
|
from dataclasses import replace
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import math
|
|
from pathlib import Path
|
|
from statistics import mean, median
|
|
from time import perf_counter
|
|
import sys
|
|
|
|
from mcm26b.config import ProblemConfig
|
|
from mcm26b.evaluation import recompute_virtual_time
|
|
from mcm26b.planner.bayes_loop import BayesLoop
|
|
from mcm26b.protocol.client import SimulatorClient
|
|
from mcm26b.sim import LocalSimulator, LocalSimTransport, Source
|
|
from mcm26b.sim.scenes import random_omni_scene
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--label', required=True)
|
|
parser.add_argument('--out', type=Path, default=Path('output/q3/optimization-20260911'))
|
|
parser.add_argument('--suite', choices=('development', 'validation'), default='development')
|
|
parser.add_argument('--baseline', action='store_true', help='加载输出根目录冻结的baseline_bayes.py及配置')
|
|
args = parser.parse_args()
|
|
folder = args.out / args.suite / args.label
|
|
folder.mkdir(parents=True, exist_ok=False)
|
|
config = ProblemConfig()
|
|
loop_cls = BayesLoop
|
|
if args.baseline:
|
|
frozen = json.loads((args.out/'baseline_config.json').read_text(encoding='utf-8'))
|
|
config = replace(config, **{name: replace(getattr(config, name), **values)
|
|
for name, values in frozen.items()})
|
|
spec = importlib.util.spec_from_file_location('mcm26b.planner._frozen_baseline',
|
|
args.out/'baseline_bayes.py')
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
loop_cls = module.BayesLoop
|
|
source_sha256 = {str(p): hashlib.sha256(p.read_bytes()).hexdigest()
|
|
for p in sorted(Path('src/mcm26b').rglob('*.py'))}
|
|
if args.baseline:
|
|
for name in ('baseline_bayes.py', 'baseline_config.json'):
|
|
source_sha256[str(args.out/name)] = hashlib.sha256((args.out/name).read_bytes()).hexdigest()
|
|
scenes = []
|
|
if args.suite == 'development':
|
|
for count in (10, 16):
|
|
for seed in range(6):
|
|
scenes.append((f'random-{count}-{seed}', random_omni_scene(seed, count), 20260910 + seed))
|
|
edge = [Source(c+1, 1800*math.cos(math.pi/6+2*math.pi*c/count),
|
|
1800*math.sin(math.pi/6+2*math.pi*c/count), 1000) for c in range(count)]
|
|
scenes.append((f'boundary-{count}', edge, 20260910))
|
|
else:
|
|
for count in (10, 13, 16):
|
|
for seed in range(100, 106):
|
|
scenes.append((f'holdout-{count}-{seed}', random_omni_scene(seed, count), 20260910+seed))
|
|
rows = []
|
|
for name, sources, error_seed in scenes:
|
|
sim = LocalSimulator(sources, robot_id='BENCH', config=config, error_seed=error_seed)
|
|
client = SimulatorClient('BENCH', config=config, transport=LocalSimTransport(sim))
|
|
start = perf_counter()
|
|
summary = loop_cls(client, config).run()
|
|
actions = client.log.action_events()
|
|
valid = (summary['completed'] and sim.cleared == {s.channel for s in sources}
|
|
and not summary['anomalies'] and summary['exit_reason'] == 'user_exit'
|
|
and not recompute_virtual_time(actions)['mismatches']
|
|
and not ({s.channel for s in sources} & set(summary['absent_channels'])))
|
|
row = dict(case=name, valid=valid, virtual_s=client.state.virtual_time_s,
|
|
measures=summary['measures'], clears=summary['clears'],
|
|
failed_clears=summary['clears']-len(sim.cleared),
|
|
distance_m=summary['move_distance_m'], wall_s=perf_counter()-start)
|
|
rows.append(row)
|
|
(folder / (name+'.json')).write_text(json.dumps(dict(
|
|
sources=[asdict(s) for s in sources], error_seed=error_seed,
|
|
summary=summary, actions=actions), ensure_ascii=False), encoding='utf-8')
|
|
print(f'{name}: valid={valid} T={row["virtual_s"]:.1f} M={row["measures"]}', flush=True)
|
|
result = dict(config=asdict(config), source='local_simulator_only', baseline=args.baseline, rows=rows,
|
|
mean_s=mean(r['virtual_s'] for r in rows), median_s=median(r['virtual_s'] for r in rows),
|
|
max_s=max(r['virtual_s'] for r in rows), valid=sum(r['valid'] for r in rows),
|
|
under_4000=sum(r['virtual_s'] < 4000 for r in rows),
|
|
source_sha256=source_sha256)
|
|
(folder/'results.json').write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8')
|
|
print({k:result[k] for k in ('mean_s','median_s','max_s','valid','under_4000')}, flush=True)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|