强化学习第二版

scripts/benchmark_rl.py

@@ -94,6 +94,9 @@ def main(argv=None) -> int:
                   for row in rows]
         cleared = all(row["rl"]["all_cleared"] and row[baseline]["all_cleared"]
                       for row in rows)
+        valid = all(part['completed'] and part['all_cleared'] and part['correct_absence']
+                    and part['normal_exit'] and not part['anomalies'] and not part['timing_mismatches']
+                    for row in rows for part in (row[baseline],row['rl']))
         report[key + "_summary"] = {
             "mean_virtual_s": {
                 baseline: statistics.mean(row[baseline]["virtual_time_s"] for row in rows),
@@ -107,6 +110,7 @@ def main(argv=None) -> int:
                 "rl": statistics.mean(row["rl"]["measures"] for row in rows),
             },
             "all_cleared": cleared,
+            "all_valid": valid,
         }
     output = Path(args.output)
     output.parent.mkdir(parents=True, exist_ok=True)
@@ -116,7 +120,7 @@ def main(argv=None) -> int:
                       "q3": report["q3_summary"],
                       "q4": report["q4_summary"]},
                      ensure_ascii=False, indent=2))
-    return 0
+    return 0 if report['q3_summary']['all_valid'] and report['q4_summary']['all_valid'] else 1
 
 
 if __name__ == "__main__":

scripts/summarize_rl_revision.py

@@ -0,0 +1,72 @@
+"""汇总配对置信区间、历史官方证据与BC/特征错位探针，不连接模拟器。"""
+from pathlib import Path
+import json
+import hashlib
+import numpy as np
+import torch
+
+import verify_rl_practice
+from mcm26b.rl.env import LocalizationEnv
+from mcm26b.rl.features import build_observation
+
+
+def main():
+    root = Path('output/rl/review-20260911')
+    records = json.loads((root/'evaluation/validation/final/results.json').read_text(encoding='utf-8'))
+    rng = np.random.default_rng(20260911)
+    comparisons = {}
+    for problem in (3,4):
+        part = [r for r in records['rows'] if r['problem']==problem]
+        for label in ('baseline','legacy'):
+            baseline = {r['case']:r for r in part if r['variant']==label}
+            selected = sorted((r for r in part if r['variant']=='revised'),key=lambda r:r['case'])
+            a = np.array([baseline[r['case']]['virtual_s'] for r in selected])
+            b = np.array([r['virtual_s'] for r in selected])
+            samples = rng.integers(0,len(a),(10000,len(a)))
+            changes = (b[samples].mean(axis=1)/a[samples].mean(axis=1)-1)*100
+            comparisons[f'q{problem}-revised-vs-{label}'] = dict(
+                relative_change_pct=(b.mean()/a.mean()-1)*100,
+                paired_bootstrap_95_pct=np.quantile(changes,[.025,.975]).tolist(),
+                faster_cases=int(sum(b<a)),runs=len(a))
+    # 重新计算原始官方日志的检查，派生报告单独保存，原始归档不写入。
+    verify_rl_practice.save_json = lambda path,value: None
+    official = [verify_rl_practice.audit(p.parent) for p in sorted(
+        Path('output/rl/official-practice').rglob('*.result.json'))]
+    env = LocalizationEnv(handoff_probability=0,boundary_probability=0)
+    fixed,_ = env.reset(seed=1)
+    # 复现旧版把整个BeliefState作为单频道对象传入的错误。
+    wrong = build_observation(env._belief.candidate_set(env._channel),env._client.state.position,
+                              env._channel,env.config,env.problem,env._blocked,env._belief,
+                              virtual_time_s=env._client.state.virtual_time_s)
+    features = dict(mismatched_indices=np.flatnonzero(fixed!=wrong).tolist(),
+                    fixed_history_count=float(fixed[7]),old_history_count=float(wrong[7]))
+    env.close()
+    torch.set_num_threads(1)
+    torch.manual_seed(0)
+    bc = {}
+    for problem in (3,4):
+        # 使用旧检查点和原始专家输入，比对原ReLU网络与拷贝后的Tanh网络。
+        checkpoint = torch.load(f'output/rl/bc/q{problem}_bc.pt',map_location='cpu',weights_only=True)
+        data = np.load(f'output/rl/bc/q{problem}_expert.npz',allow_pickle=False)
+        x = torch.from_numpy(data['obs'][::max(1,len(data['obs'])//1000)].copy())
+        def network(activation):
+            net = torch.nn.Sequential(torch.nn.Linear(401,128),activation(),
+                                      torch.nn.Linear(128,128),activation(),torch.nn.Linear(128,60))
+            net.load_state_dict({k.removeprefix('net.'):v for k,v in checkpoint['state_dict'].items()})
+            return net
+        with torch.no_grad():
+            before,after=network(torch.nn.ReLU)(x),network(torch.nn.Tanh)(x)
+        bc[f'q{problem}'] = dict(samples=len(x),
+            different_argmax_fraction=float((before.argmax(1)!=after.argmax(1)).float().mean()),
+            max_logit_difference=float((before-after).abs().max()))
+    report = dict(comparisons=comparisons,official=official,feature_probe=features,bc_probe=bc,
+                  limitations='置信区间仅重采样固定模型的场景，不覆盖训练随机种子不确定性。',
+                  code_sha256={str(p):hashlib.sha256(p.read_bytes()).hexdigest()
+                               for p in sorted(Path('src/mcm26b/rl').glob('*.py'))})
+    (root/'review_evidence.json').write_text(json.dumps(report,ensure_ascii=False,indent=2),encoding='utf-8')
+    print(json.dumps(dict(comparisons=comparisons,feature_probe=features,bc_probe=bc,
+                         official=[dict(case=r['case_code'],passed=r['passed']) for r in official]),indent=2))
+
+
+if __name__=='__main__':
+    main()

scripts/verify_rl_practice.py

@@ -35,7 +35,7 @@ def audit(folder: Path) -> dict:
 
     checks["official_problem_matches_summary"] = (
         official["problem_no"] in (3, 4)
-        and summary["strategy"] in ("rl", "rl_q4")
+        and summary["strategy"] == ("rl" if official["problem_no"] == 3 else "rl_q4")
         and 10 <= official["jammer_count"] <= 16
         and official["omnidirectional_jammer_count"] + official["directional_jammer_count"]
         == official["jammer_count"])