13

scripts/benchmark_rl.py

@@ -18,6 +18,7 @@ import time
 from pathlib import Path
 
 from mcm26b.config import ProblemConfig
+from mcm26b.evaluation import recompute_virtual_time
 from mcm26b.harness import (LocalSimTransport, mixed_scene, random_omni_scene,
                             referee_spec)
 from mcm26b.planner import DirectionalExtension
@@ -36,8 +37,17 @@ def run_one(loop_cls, sources, *, config, robot_id: str, **kwargs) -> dict:
     started = time.monotonic()
     summary = loop_cls(client, config, **kwargs).run() \
         if kwargs else loop_cls(client, config).run()
+    events = client.log.action_events()
+    successful = {e['request']['channel'] for e in events
+                  if e['response'].get('clear_result') == 'success'}
+    truth = {s.channel for s in sources}
     return {
         "completed": bool(summary["completed"]),
+        "all_cleared": successful == truth,
+        "correct_absence": not bool(set(summary['absent_channels']) & truth),
+        "anomalies": summary['anomalies'],
+        "timing_mismatches": len(recompute_virtual_time(events)['mismatches']),
+        "normal_exit": summary['exit_reason'] == 'user_exit',
         "virtual_time_s": float(summary["virtual_time_s"]),
         "measures": int(summary["measures"]),
         "total_requests": int(summary["total_requests"]),
@@ -82,7 +92,7 @@ def main(argv=None) -> int:
         rows = report[key]
         deltas = [row["rl"]["virtual_time_s"] - row[baseline]["virtual_time_s"]
                   for row in rows]
-        cleared = all(row["rl"]["completed"] and row[baseline]["completed"]
+        cleared = all(row["rl"]["all_cleared"] and row[baseline]["all_cleared"]
                       for row in rows)
         report[key + "_summary"] = {
             "mean_virtual_s": {