13

scripts/benchmark_rl.py

@@ -18,6 +18,7 @@ import time
 from pathlib import Path
 
 from mcm26b.config import ProblemConfig
+from mcm26b.evaluation import recompute_virtual_time
 from mcm26b.harness import (LocalSimTransport, mixed_scene, random_omni_scene,
                             referee_spec)
 from mcm26b.planner import DirectionalExtension
@@ -36,8 +37,17 @@ def run_one(loop_cls, sources, *, config, robot_id: str, **kwargs) -> dict:
     started = time.monotonic()
     summary = loop_cls(client, config, **kwargs).run() \
         if kwargs else loop_cls(client, config).run()
+    events = client.log.action_events()
+    successful = {e['request']['channel'] for e in events
+                  if e['response'].get('clear_result') == 'success'}
+    truth = {s.channel for s in sources}
     return {
         "completed": bool(summary["completed"]),
+        "all_cleared": successful == truth,
+        "correct_absence": not bool(set(summary['absent_channels']) & truth),
+        "anomalies": summary['anomalies'],
+        "timing_mismatches": len(recompute_virtual_time(events)['mismatches']),
+        "normal_exit": summary['exit_reason'] == 'user_exit',
         "virtual_time_s": float(summary["virtual_time_s"]),
         "measures": int(summary["measures"]),
         "total_requests": int(summary["total_requests"]),
@@ -82,7 +92,7 @@ def main(argv=None) -> int:
         rows = report[key]
         deltas = [row["rl"]["virtual_time_s"] - row[baseline]["virtual_time_s"]
                   for row in rows]
-        cleared = all(row["rl"]["completed"] and row[baseline]["completed"]
+        cleared = all(row["rl"]["all_cleared"] and row[baseline]["all_cleared"]
                       for row in rows)
         report[key + "_summary"] = {
             "mean_virtual_s": {

scripts/evaluate_rl_revision.py

@@ -0,0 +1,117 @@
+"""同场景配对比较手写策略、旧PPO及修订模型；真值仅在评测端使用。"""
+import argparse
+from dataclasses import asdict
+import hashlib
+import json
+import math
+from pathlib import Path
+from statistics import mean, median
+from time import perf_counter
+
+import torch
+
+from mcm26b.config import ProblemConfig
+from mcm26b.evaluation import recompute_virtual_time
+from mcm26b.harness import LocalSimTransport, mixed_scene, random_omni_scene, referee_spec
+from mcm26b.planner import DirectionalExtension
+from mcm26b.planner.bayes_loop import BayesLoop
+from mcm26b.planner.rl_loop import RLBayesLoop, RLDirectionalLoop
+from mcm26b.protocol.client import SimulatorClient
+from referee import LocalSimulator, Source
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--split', choices=('development','validation','stress'), default='development')
+    parser.add_argument('--variants', default='baseline,legacy,revised')
+    parser.add_argument('--problems', default='3,4')
+    parser.add_argument('--model-dir', type=Path, default=Path('output/rl/review-20260911'))
+    parser.add_argument('--label', required=True)
+    args = parser.parse_args()
+    torch.set_num_threads(1)
+    config = ProblemConfig()
+    output = args.model_dir / 'evaluation' / args.split / args.label
+    output.mkdir(parents=True, exist_ok=False)
+    variants = args.variants.split(',')
+    rows = []
+    def save():
+        aggregates = {}
+        for problem in (3,4):
+            for variant in variants:
+                part = [r for r in rows if r['problem']==problem and r['variant']==variant]
+                if part:
+                    aggregates[f'q{problem}-{variant}'] = dict(
+                        runs=len(part), valid=sum(r['valid'] for r in part),
+                        mean_virtual_s=mean(r['virtual_s'] for r in part),
+                        median_virtual_s=median(r['virtual_s'] for r in part),
+                        max_virtual_s=max(r['virtual_s'] for r in part),
+                        failed_clears=sum(r['failed_clears'] for r in part),
+                        repeated_measures=sum(r['repeated_measures'] for r in part))
+        result = dict(source='local_referee_only', config=asdict(config), split=args.split,
+                      rows=rows, aggregates=aggregates)
+        (output/'results.json').write_text(json.dumps(result,ensure_ascii=False,indent=2),encoding='utf-8')
+        return aggregates
+    for problem in map(int, args.problems.split(',')):
+        scenes = []
+        if args.split == 'stress':
+            for count in (10,16):
+                scenes.append((f'boundary-{count}', count, 99000+count, [
+                    Source(c+1,1800*math.cos(2*math.pi*c/count),1800*math.sin(2*math.pi*c/count),
+                           1000,directional=(problem==4 and c%2==0),
+                           heading_deg=360*c/count if problem==4 and c%2==0 else 0)
+                    for c in range(count)]))
+        else:
+            seeds = range(700,703) if args.split=='development' else range(91000,91006)
+            counts = (10,16) if args.split=='development' else (10,13,16)
+            for count in counts:
+                for seed in seeds:
+                    directional = (1, count//2, count-1)[seed%3]
+                    sources = (random_omni_scene(seed, count, config) if problem==3 else
+                               mixed_scene(seed, count, directional, config))
+                    scenes.append((f'{count}-{seed}',count,seed,sources))
+        for name,count,seed,sources in scenes:
+            scene_data = dict(sources=[asdict(s) for s in sources], error_seed=seed+2000000)
+            (output/f'q{problem}-{name}-scene.json').write_text(json.dumps(scene_data),encoding='utf-8')
+            for variant in variants:
+                model = None
+                cls = BayesLoop if problem==3 else DirectionalExtension
+                kwargs = {}
+                if variant != 'baseline':
+                    cls = RLBayesLoop if problem==3 else RLDirectionalLoop
+                    model = (Path(f'output/rl/q{problem}_ppo.zip') if variant=='legacy'
+                             else args.model_dir/f'q{problem}_masked.zip')
+                    kwargs = dict(model_path=model, device='cpu')
+                sim = LocalSimulator(sources,robot_id='RL-AUDIT',spec=referee_spec(config),error_seed=seed+2000000)
+                client = SimulatorClient('RL-AUDIT',config=config,transport=LocalSimTransport(sim))
+                start = perf_counter()
+                summary = cls(client,config,**kwargs).run()
+                events = client.log.action_events()
+                truth = {s.channel for s in sources}
+                successful = {e['request']['channel'] for e in events if e['response'].get('clear_result')=='success'}
+                seen = set()
+                repeats = 0
+                for e in events:
+                    if e['path']=='/measure':
+                        p=e['request']['position']
+                        tag=(e['request']['channel'],round(p['x'],6),round(p['y'],6))
+                        repeats += tag in seen
+                        seen.add(tag)
+                mismatches = recompute_virtual_time(events)['mismatches']
+                valid = (successful==truth and summary['completed'] and not summary['anomalies']
+                         and not set(summary['absent_channels']) & truth
+                         and summary['exit_reason']=='user_exit' and not mismatches)
+                row = dict(case=name,problem=problem,variant=variant,valid=valid,
+                           virtual_s=client.state.virtual_time_s,all_cleared=successful==truth,
+                           measures=summary['measures'],failed_clears=summary['clears']-len(successful),
+                           repeated_measures=repeats,wall_s=perf_counter()-start,
+                           model_sha256=hashlib.sha256(model.read_bytes()).hexdigest() if model else None)
+                rows.append(row)
+                (output/f'q{problem}-{name}-{variant}.json').write_text(json.dumps(
+                    dict(row=row,summary=summary,events=events,timing_mismatches=mismatches),ensure_ascii=False),encoding='utf-8')
+                save()
+                print(json.dumps(row),flush=True)
+    print(json.dumps(save(),indent=2),flush=True)
+
+
+if __name__=='__main__':
+    main()

scripts/pretrain_rl_expert.py

@@ -73,8 +73,8 @@ class BCMlp(torch.nn.Module):
     def __init__(self):
         super().__init__()
         self.net = torch.nn.Sequential(
-            torch.nn.Linear(OBS_DIM, HIDDEN[0]), torch.nn.ReLU(),
-            torch.nn.Linear(HIDDEN[0], HIDDEN[1]), torch.nn.ReLU(),
+            torch.nn.Linear(OBS_DIM, HIDDEN[0]), torch.nn.Tanh(),
+            torch.nn.Linear(HIDDEN[0], HIDDEN[1]), torch.nn.Tanh(),
             torch.nn.Linear(HIDDEN[1], NUM_ACTIONS))
 
     def forward(self, x):
@@ -86,30 +86,33 @@ def collect(problem: int, episodes: int, seed: int, config: ProblemConfig):
         config, problem=problem,
         directional_probability=1.0 if problem == 4 else 0.0,
         max_episode_steps=16)
-    observations, actions = [], []
+    observations, actions, groups = [], [], []
     for episode in range(episodes):
         obs, info = env.reset(seed=seed + episode)
         for _step in range(env.max_episode_steps + 2):
             action = expert_action(env)
             observations.append(obs)
             actions.append(action)
+            groups.append(episode)
             obs, _reward, terminated, truncated, _info = env.step(action)
             if terminated or truncated:
                 break
     return np.asarray(observations, dtype=np.float32), \
-        np.asarray(actions, dtype=np.int64)
+        np.asarray(actions, dtype=np.int64), np.asarray(groups, dtype=np.int64)
 
 
 def train_bc(x, y, *, epochs: int, batch_size: int, lr: float,
-             device: str, valid_frac: float = 0.1):
+             device: str, groups, valid_frac: float = 0.1):
     model = BCMlp().to(device)
     optimizer = torch.optim.AdamW(model.parameters(), lr=lr)
-    count = len(x)
-    split = max(1, int(count * valid_frac))
-    perm = torch.randperm(count)
+    episodes = np.unique(groups)
+    if len(episodes) < 2:
+        raise ValueError('行为克隆至少需要两个独立episode才能划分验证集')
+    valid_groups = np.random.default_rng(0).permutation(episodes)[:max(1, int(len(episodes)*valid_frac))]
+    valid = torch.as_tensor(np.isin(groups, valid_groups), device=device)
     x_t, y_t = torch.as_tensor(x, device=device), torch.as_tensor(y, device=device)
-    x_v, y_v = x_t[perm[:split]], y_t[perm[:split]]
-    x_tr, y_tr = x_t[perm[split:]], y_t[perm[split:]]
+    x_v, y_v = x_t[valid], y_t[valid]
+    x_tr, y_tr = x_t[~valid], y_t[~valid]
     loss_fn = torch.nn.CrossEntropyLoss()
     for epoch in range(epochs):
         order = torch.randperm(len(x_tr), device=device)
@@ -137,10 +140,10 @@ def train_bc(x, y, *, epochs: int, batch_size: int, lr: float,
 
 def copy_into_ppo(problem: int, model: BCMlp, device: str) -> Path | None:
     """把 BC 网络权重按 SB3 的 policy_net/action_net 拆分拷入 PPO。"""
-    ppo_path = Path(f"output/rl/q{problem}_ppo.zip")
-    if not ppo_path.is_file():
-        return None
-    ppo = PPO.load(str(ppo_path), device=device)
+    # 当前奖励/特征已修订，重新初始化critic，不能沿用旧任务的价值预测。
+    env = LocalizationEnv(problem=problem)
+    ppo = PPO('MlpPolicy', env, policy_kwargs={'net_arch': list(HIDDEN)},
+              n_steps=512, batch_size=128, gamma=1.0, device=device)
     state = {key.removeprefix("net."): value
              for key, value in model.state_dict().items()}
     # SB3 MlpPolicy：policy_net 只含前两个 Linear（Tanh 激活），
@@ -151,8 +154,17 @@ def copy_into_ppo(problem: int, model: BCMlp, device: str) -> Path | None:
         "weight": state["4.weight"], "bias": state["4.bias"]}
     ppo.policy.mlp_extractor.policy_net.load_state_dict(policy_state)
     ppo.policy.action_net.load_state_dict(action_state)
+    # 数值检查整个前向映射，而不只是线性层形状。
+    probe = torch.randn((8, OBS_DIM), device=device)
+    with torch.no_grad():
+        copied = ppo.policy.action_net(ppo.policy.mlp_extractor.policy_net(probe))
+        if not torch.allclose(model(probe), copied, atol=1e-6):
+            raise RuntimeError('行为克隆与PPO激活函数/前向映射不一致')
     init_path = Path(f"output/rl/q{problem}_ppo_bc_init.zip")
     ppo.save(str(init_path))
+    env.close()
+    init_path.with_suffix('.json').write_text(json.dumps(
+        dict(algorithm='ppo', feature_version=2, problem=problem)), encoding='utf-8')
     return init_path
 
 
@@ -182,6 +194,8 @@ def main(argv=None) -> int:
         bc_dir = Path("output/rl/bc")
         checkpoint = torch.load(bc_dir / f"q{args.problem}_bc.pt",
                                 map_location="cpu")
+        if checkpoint.get('activation') != 'Tanh' or checkpoint.get('feature_version') != 2:
+            parser.error('旧ReLU/旧特征BC检查点不能直接拷贝，请重新采集并训练')
         model = BCMlp()
         model.load_state_dict(checkpoint["state_dict"])
         init_path = copy_into_ppo(args.problem, model, device)
@@ -204,17 +218,19 @@ def main(argv=None) -> int:
 
     config = ProblemConfig()
     started = time.monotonic()
-    x, y = collect(args.problem, args.episodes, args.seed, config)
+    torch.manual_seed(args.seed)
+    x, y, groups = collect(args.problem, args.episodes, args.seed, config)
     collect_s = time.monotonic() - started
     bc_dir = Path("output/rl/bc")
     bc_dir.mkdir(parents=True, exist_ok=True)
-    np.savez_compressed(bc_dir / f"q{args.problem}_expert.npz", obs=x, action=y)
+    np.savez_compressed(bc_dir / f"q{args.problem}_expert.npz", obs=x, action=y, episode=groups)
 
     model, valid_acc = train_bc(x, y, epochs=args.epochs,
                                 batch_size=args.batch_size, lr=args.lr,
-                                device=device)
+                                device=device, groups=groups)
     torch.save({"state_dict": model.state_dict(), "obs_dim": OBS_DIM,
-                "actions": NUM_ACTIONS, "valid_acc": valid_acc},
+                "actions": NUM_ACTIONS, "valid_acc": valid_acc,
+                "activation": "Tanh", "feature_version": 2},
                bc_dir / f"q{args.problem}_bc.pt")
 
     # 拷入现有 PPO 的 policy_net/action_net 作为初始化（critic 保持原权重）。

scripts/train_rl.py

@@ -17,9 +17,12 @@ from __future__ import annotations
 import argparse
 import json
 import time
+from dataclasses import asdict
+import hashlib
 from pathlib import Path
 
 from stable_baselines3 import PPO
+from sb3_contrib import MaskablePPO
 from stable_baselines3.common.vec_env import DummyVecEnv
 
 from mcm26b.config import ProblemConfig
@@ -46,11 +49,13 @@ def evaluate(model, problem: int, *, episodes: int = 24,
     total_reward = 0.0
     successes = 0
     steps = []
+    costs = []
     for episode in range(episodes):
         obs, _info = env.reset(seed=1_000_000 + episode)
         episode_reward = 0.0
         for step in range(env.max_episode_steps + 2):
-            action, _state = model.predict(obs, deterministic=True)
+            kwargs = {'action_masks': env.action_masks()} if isinstance(model, MaskablePPO) else {}
+            action, _state = model.predict(obs, deterministic=True, **kwargs)
             obs, reward, terminated, truncated, info = env.step(int(action))
             episode_reward += float(reward)
             if terminated or truncated:
@@ -59,12 +64,16 @@ def evaluate(model, problem: int, *, episodes: int = 24,
         cleared = bool(info.get("cleared"))
         successes += int(cleared)
         steps.append(step + 1)
+        costs.append(info['virtual_cost_s'])
+    env.close()
     return {
         "episodes": episodes,
         "success_rate": successes / episodes,
         "mean_steps": sum(steps) / episodes,
         "mean_episode_reward": total_reward / episodes,
         "successes": successes,
+        "mean_virtual_cost_s": sum(costs) / episodes,
+        "max_virtual_cost_s": max(costs),
     }
 
 
@@ -87,6 +96,8 @@ def main(argv=None) -> int:
                              "加载后继续 PPO 微调）")
     parser.add_argument("--eval-episodes", type=int, default=24)
     parser.add_argument("--verbose", type=int, default=1)
+    parser.add_argument('--algorithm', choices=('ppo', 'maskable_ppo'), default='maskable_ppo')
+    parser.add_argument('--torch-threads', type=int, default=1)
     args = parser.parse_args(argv)
     directional_probability = (args.directional_probability
                                if args.directional_probability is not None
@@ -95,6 +106,7 @@ def main(argv=None) -> int:
         parser.error("--directional-probability 须在 [0,1]")
 
     import torch
+    torch.set_num_threads(args.torch_threads)
     if args.device == "auto":
         device = "cuda" if torch.cuda.is_available() else "cpu"
     else:
@@ -111,20 +123,24 @@ def main(argv=None) -> int:
     }, ensure_ascii=False))
 
     config = ProblemConfig()
+    model_cls = MaskablePPO if args.algorithm == 'maskable_ppo' else PPO
     vec_env = DummyVecEnv(
         [_make_env(args.problem, args.seed + rank, config,
                    directional_probability)
          for rank in range(max(1, args.envs))])
     if args.init:
-        model = PPO.load(args.init, env=vec_env, device=device)
+        metadata = json.loads(Path(args.init).with_suffix('.json').read_text(encoding='utf-8'))
+        if metadata.get('algorithm', 'ppo') != args.algorithm or metadata.get('feature_version') != 2:
+            parser.error('初始化模型的算法/特征版本与当前训练不符，请重训或提供v2初始化模型')
+        model = model_cls.load(args.init, env=vec_env, device=device)
     else:
-        model = PPO(
+        model = model_cls(
             "MlpPolicy", vec_env,
             policy_kwargs={"net_arch": [128, 128]},
             n_steps=512,
             batch_size=128,
             n_epochs=6,
-            gamma=0.99,
+            gamma=1.0,
             gae_lambda=0.95,
             learning_rate=3e-4,
             clip_range=0.2,
@@ -158,12 +174,19 @@ def main(argv=None) -> int:
         "wall_seconds": round(elapsed_s, 1),
         "model": str(model_path),
         "init": args.init,
+        "algorithm": args.algorithm,
+        "feature_version": 2,
+        "reward": 'negative_virtual_cost_div100_success20_failure100',
+        "gamma": 1.0,
+        "config": asdict(config),
+        "model_sha256": hashlib.sha256(model_path.read_bytes()).hexdigest(),
         **evaluation,
     }
     report_path = model_path.with_suffix(".json")
     report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2),
                            encoding="utf-8")
     print(json.dumps(report, ensure_ascii=False, indent=2))
+    vec_env.close()
     return 0