114514

scripts/ablate_hrl.py

@@ -0,0 +1,25 @@
+"""HRL 消融入口（P2 待实现）。
+
+预定义消融维度：时间目标 vs prompt 塑形、信息增益奖励系数、动作掩码取消范围、
+课程学习、粒子统计特征。硬性协议守卫/证书/预算永远保留，不在消融中取消。
+"""
+from __future__ import annotations
+
+import argparse
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--variants", default="time,prompt,no_info,no_mask,no_curriculum")
+    parser.add_argument("--manifest", default="output/hrl/manifests/final.json")
+    parser.add_argument("--output", default="output/hrl/p2-ablation")
+    parser.add_argument("--timesteps", type=int, default=2048)
+    parser.add_argument("--seeds", type=int, default=5)
+    parser.parse_args(argv)
+    raise NotImplementedError(
+        "P2 尚未实现：必须先完成专家 BC 与冻结场景对照，"
+        "再对相同训练步数/场景/种子逐项消融；P1 只保证 benchmark/training/CLI。")
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/benchmark_hrl.py

@@ -0,0 +1,429 @@
+"""HRL/基线统一评测入口（P1）。
+
+特点：
+- 同一 manifest 中的真值场景被所有 variant 复用，保证配对比较；
+- 真值仅在测试端用于构造 ``LocalSimulator`` 与最终核对，策略端只收到协议响应；
+- 支持 ``hrl_heuristic``（高层启发式降级）、``bayes``、``directional``；
+- 支持已训练高层模型 ``hrl``（需 ``--hrl-model`` 或 manifest 中的 variant 配置）。
+
+输出目录（默认拒绝覆盖已有结果）：
+    <output>/manifest.json
+    <output>/runs/<scene_id>/<variant>/summary.json
+    <output>/runs/<scene_id>/<variant>/session.jsonl
+    <output>/results.json
+    <output>/aggregate.json
+    <output>/aggregate.csv
+
+示例：
+    python scripts/benchmark_hrl.py \
+        --variants hrl_heuristic,bayes,directional \
+        --manifest output/hrl/manifests/dev.json \
+        --output output/hrl/p1-heuristic
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import math
+import statistics
+import sys
+import time
+from collections import defaultdict
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Any, Callable
+
+from mcm26b.config import ProblemConfig
+from mcm26b.harness import LocalSimTransport, mixed_scene, random_omni_scene
+from mcm26b.harness.scenes import referee_spec
+from mcm26b.protocol import SimulatorClient
+from mcm26b.protocol.log import SessionLog
+
+try:  # 测试端依赖；运行时代码不导入
+    from referee import LocalSimulator, Source
+except ImportError as exc:  # pragma: no cover
+    raise SystemExit("benchmark_hrl 需要 dev 依赖 referee。请先 `uv sync`。") from exc
+
+
+# ---------------------------------------------------------------------------
+# manifest / scene
+# ---------------------------------------------------------------------------
+@dataclass(frozen=True)
+class SceneSpec:
+    scene_id: str
+    seed: int
+    error_seed: int
+    problem: int
+    sources: list[dict[str, Any]]
+
+    @property
+    def truth_channels(self) -> tuple[int, ...]:
+        return tuple(sorted(int(src["channel"]) for src in self.sources))
+
+
+def _source_dict(source: Source) -> dict[str, Any]:
+    return {
+        "channel": int(source.channel),
+        "x": float(source.x),
+        "y": float(source.y),
+        "r_eff_m": float(source.r_eff_m),
+        "directional": bool(source.directional),
+        "heading_deg": float(source.heading_deg),
+    }
+
+
+def _sources_to_dicts(sources: list[Source]) -> list[dict[str, Any]]:
+    return [_source_dict(source) for source in sources]
+
+
+def _source_from_dict(data: dict[str, Any]) -> Source:
+    return Source(
+        channel=int(data["channel"]),
+        x=float(data["x"]),
+        y=float(data["y"]),
+        r_eff_m=float(data["r_eff_m"]),
+        directional=bool(data.get("directional", False)),
+        heading_deg=float(data.get("heading_deg", 0.0)),
+    )
+
+
+def make_default_manifest(*, problem: int, scenes: int, sources: int | None,
+                          directional: int, start_seed: int,
+                          config: ProblemConfig) -> dict[str, Any]:
+    """按 CLI 参数生成可复现的固定场景清单。"""
+    entries: list[dict[str, Any]] = []
+    for index in range(scenes):
+        seed = start_seed + index
+        if problem == 4:
+            scene_sources = mixed_scene(seed=seed, count=sources,
+                                        directional=directional, config=config)
+        else:
+            scene_sources = random_omni_scene(seed=seed, count=sources,
+                                              config=config)
+        entries.append({
+            "scene_id": f"scene-{seed}",
+            "seed": seed,
+            "error_seed": seed + 10_000,
+            "problem": problem,
+            "sources": _sources_to_dicts(scene_sources),
+        })
+    return {"version": 1, "problem": problem, "scenes": entries}
+
+
+def load_or_generate_manifest(path: Path, *, problem: int, scenes: int,
+                              sources: int | None, directional: int,
+                              start_seed: int,
+                              config: ProblemConfig) -> dict[str, Any]:
+    if path.exists():
+        manifest = json.loads(path.read_text(encoding="utf-8"))
+    else:
+        manifest = make_default_manifest(
+            problem=problem, scenes=scenes, sources=sources,
+            directional=directional, start_seed=start_seed, config=config)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2),
+                        encoding="utf-8")
+    if "scenes" not in manifest or not manifest["scenes"]:
+        raise ValueError(f"manifest 无 scenes: {path}")
+    for raw in manifest["scenes"]:
+        if not {"scene_id", "error_seed", "problem", "sources"} <= set(raw):
+            raise ValueError(f"manifest scene 字段缺失: {raw!r}")
+        if not raw["sources"]:
+            raise ValueError(f"scene {raw.get('scene_id')} 没有来源真值")
+    return manifest
+
+
+def scenes_from_manifest(manifest: dict[str, Any]) -> list[SceneSpec]:
+    result: list[SceneSpec] = []
+    for raw in manifest["scenes"]:
+        result.append(SceneSpec(
+            scene_id=str(raw["scene_id"]),
+            seed=int(raw.get("seed", 0)),
+            error_seed=int(raw["error_seed"]),
+            problem=int(raw["problem"]),
+            sources=[dict(item) for item in raw["sources"]],
+        ))
+    return result
+
+
+# ---------------------------------------------------------------------------
+# variants
+# ---------------------------------------------------------------------------
+def _build_loop(variant: str, client: SimulatorClient, config: ProblemConfig,
+                problem: int, *, hrl_model: str | None,
+                hrl_device: str = "auto"):
+    """延迟导入变体实现，避免 benchmark 顶层拉起重依赖。"""
+    if variant == "bayes":
+        from mcm26b.planner.bayes_loop import BayesLoop
+        return BayesLoop(client, config)
+    if variant == "directional":
+        from mcm26b.planner import DirectionalExtension
+        return DirectionalExtension(client, config)
+    if variant == "hrl_heuristic":
+        from mcm26b.planner.hrl_loop import HRLLoop
+        return HRLLoop(client, config, problem=problem, high_policy=None)
+    if variant == "hrl":
+        if not hrl_model:
+            raise RuntimeError("variant=hrl 需要 --hrl-model")
+        from mcm26b.hrl.policy import HighLevelPolicy
+        from mcm26b.planner.hrl_loop import HRLLoop
+        policy = HighLevelPolicy(hrl_model, device=hrl_device, problem=problem)
+        return HRLLoop(client, config, problem=problem, high_policy=policy)
+    raise ValueError(f"未知 variant: {variant!r}")
+
+
+def _validity(summary: dict[str, Any], truth_channels: tuple[int, ...]) -> dict[str, Any]:
+    """测试端有效性审计。真值只在此处进入。"""
+    cleared = set(int(c) for c in summary.get("cleared_channels", []))
+    absent = set(int(c) for c in summary.get("absent_channels", []))
+    truth = set(truth_channels)
+    declared_absent = truth & absent
+    cleared_matches_truth = cleared == truth
+    no_source_declared_absent = not declared_absent
+    completed = bool(summary.get("completed"))
+    anomalies = list(summary.get("anomalies", []))
+    # 本地 benchmark 要求：清洗真值且无守恒性异常；缺席集合不得含真值。
+    valid = completed and cleared_matches_truth and no_source_declared_absent \
+        and not anomalies
+    return {
+        "valid": valid,
+        "completed": completed,
+        "cleared_matches_truth": cleared_matches_truth,
+        "no_source_declared_absent": no_source_declared_absent,
+        "truth_channels": sorted(truth),
+        "declared_absent_truth": sorted(declared_absent),
+        "anomaly_count": len(anomalies),
+    }
+
+
+def _row_from_summary(*, scene: SceneSpec, variant: str, summary: dict[str, Any],
+                      wall_s: float) -> dict[str, Any]:
+    validity = _validity(summary, scene.truth_channels)
+    cleared_count = len(summary.get("cleared_channels", []))
+    virtual_s = float(summary.get("virtual_time_s") or 0.0)
+    avg_s = summary.get("avg_locate_clear_time_s")
+    totals = summary.get("step_totals", {}) or {}
+    return {
+        "scene_id": scene.scene_id,
+        "variant": variant,
+        "problem": scene.problem,
+        "truth_count": len(scene.truth_channels),
+        "cleared_count": cleared_count,
+        "virtual_time_s": virtual_s,
+        "avg_locate_clear_time_s": (float(avg_s) if avg_s is not None else None),
+        "accounted_virtual_time_s": summary.get("accounted_virtual_time_s"),
+        "program_run_time_s": round(wall_s, 6),
+        "real_elapsed_total_s": totals.get("real_elapsed_total_s"),
+        "total_requests": summary.get("total_requests"),
+        "measures": summary.get("measures"),
+        "clears": summary.get("clears"),
+        "clear_success": summary.get("clear_success"),
+        "completed": bool(summary.get("completed")),
+        "stop_reason": summary.get("stop_reason"),
+        "exit_reason": summary.get("exit_reason"),
+        **validity,
+    }
+
+
+def _error_row(scene: SceneSpec, variant: str, exc: BaseException) -> dict[str, Any]:
+    return {
+        "scene_id": scene.scene_id,
+        "variant": variant,
+        "problem": scene.problem,
+        "truth_count": len(scene.truth_channels),
+        "cleared_count": 0,
+        "virtual_time_s": None,
+        "avg_locate_clear_time_s": None,
+        "accounted_virtual_time_s": None,
+        "program_run_time_s": None,
+        "real_elapsed_total_s": None,
+        "total_requests": None,
+        "measures": None,
+        "clears": None,
+        "clear_success": None,
+        "completed": False,
+        "stop_reason": None,
+        "exit_reason": None,
+        "valid": False,
+        "cleared_matches_truth": False,
+        "no_source_declared_absent": False,
+        "truth_channels": sorted(scene.truth_channels),
+        "declared_absent_truth": [],
+        "anomaly_count": 0,
+        "error": f"{type(exc).__name__}: {exc}",
+    }
+
+
+def run_scene_variant(scene: SceneSpec, variant: str, *, config: ProblemConfig,
+                      hrl_model: str | None, run_dir: Path,
+                      hrl_device: str = "auto") -> dict[str, Any]:
+    """运行一个 scene × variant。返回可序列化结果行与 summary。"""
+    run_dir.mkdir(parents=True, exist_ok=True)
+    sources = [_source_from_dict(raw) for raw in scene.sources]
+    simulator = LocalSimulator(
+        sources,
+        robot_id="hrl-benchmark",
+        spec=referee_spec(config),
+        error_seed=scene.error_seed,
+        stress_mode=False,
+    )
+    # 真值只用于 LocalSimulator；策略端仅持有 client/loop，不读取 simulator.sources。
+    log_path = run_dir / "session.jsonl"
+    log = SessionLog(path=log_path)
+    client = SimulatorClient("hrl-benchmark", config=config,
+                             transport=LocalSimTransport(simulator), log=log)
+    loop = _build_loop(variant, client, config, scene.problem,
+                       hrl_model=hrl_model, hrl_device=hrl_device)
+    started = time.monotonic()
+    try:
+        summary = loop.run()
+    finally:
+        wall_s = time.monotonic() - started
+    row = _row_from_summary(scene=scene, variant=variant, summary=summary,
+                            wall_s=wall_s)
+    # 完整 summary 可能很大；仍写入 runs 目录，供复算与异常审计。
+    (run_dir / "summary.json").write_text(
+        json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
+    return {"row": row, "summary": summary}
+
+
+# ---------------------------------------------------------------------------
+# aggregate
+# ---------------------------------------------------------------------------
+def _aggregate(rows: list[dict[str, Any]]) -> dict[str, Any]:
+    grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
+    for row in rows:
+        grouped[row["variant"]].append(row)
+    result: dict[str, Any] = {"variants": {}, "all_valid": True}
+    for variant, items in sorted(grouped.items()):
+        virtual = [float(item["virtual_time_s"]) for item in items
+                   if item.get("virtual_time_s") is not None]
+        per_source = [float(item["avg_locate_clear_time_s"])
+                      for item in items
+                      if item.get("avg_locate_clear_time_s") is not None]
+        cleared = [int(item["cleared_count"] or 0) for item in items]
+        program = [float(item["program_run_time_s"]) for item in items
+                   if item.get("program_run_time_s") is not None]
+        success = [1.0 if item.get("valid") else 0.0 for item in items]
+        result["variants"][variant] = {
+            "episodes": len(items),
+            "valid_episodes": int(sum(success)),
+            "valid_rate": (sum(success) / len(items)) if items else 0.0,
+            "success_rate": (sum(success) / len(items)) if items else 0.0,
+            "cleared_count_mean": statistics.fmean(cleared) if cleared else 0.0,
+            "cleared_count_std": (statistics.pstdev(cleared)
+                                  if len(cleared) > 1 else 0.0),
+            "virtual_time_mean_s": statistics.fmean(virtual) if virtual else 0.0,
+            "virtual_time_std_s": (statistics.pstdev(virtual)
+                                   if len(virtual) > 1 else 0.0),
+            "virtual_time_min_s": min(virtual) if virtual else 0.0,
+            "virtual_time_max_s": max(virtual) if virtual else 0.0,
+            "avg_locate_clear_time_mean_s": (statistics.fmean(per_source)
+                                             if per_source else None),
+            "program_run_time_mean_s": (statistics.fmean(program)
+                                        if program else None),
+            "total_requests_mean": statistics.fmean(
+                [int(item.get("total_requests") or 0) for item in items]),
+        }
+        if result["variants"][variant]["valid_rate"] < 1.0:
+            result["all_valid"] = False
+    return result
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--variants", default="hrl_heuristic,bayes,directional",
+                        help="逗号分隔：hrl_heuristic,bayes,directional,hrl")
+    parser.add_argument("--manifest", default="output/hrl/manifests/dev.json")
+    parser.add_argument("--output", default="output/hrl/p1-benchmark")
+    parser.add_argument("--problem", type=int, choices=(3, 4), default=3)
+    parser.add_argument("--scenes", type=int, default=6,
+                        help="manifest 不存在时生成的场景数")
+    parser.add_argument("--sources", type=int, default=None,
+                        help="每局源数；默认按题面 10~16 随机")
+    parser.add_argument("--directional", type=int, default=3,
+                        help="问题4定向源数量（生成 manifest 时使用）")
+    parser.add_argument("--start-seed", type=int, default=91_000)
+    parser.add_argument("--hrl-model", default=None)
+    parser.add_argument("--hrl-device", choices=("auto", "cpu", "cuda", "mps"),
+                        default="auto")
+    parser.add_argument("--force", action="store_true",
+                        help="允许覆盖已有输出目录")
+    parser.add_argument("--log", action="store_true",
+                        help="为兼容 CLI 保留；当前始终写 session.jsonl")
+    args = parser.parse_args(argv)
+
+    config = ProblemConfig()
+    output = Path(args.output)
+    if output.exists() and any(output.iterdir()) and not args.force:
+        print(f"输出目录已存在且非空：{output}；如需覆盖请加 --force。",
+              file=sys.stderr)
+        return 2
+    output.mkdir(parents=True, exist_ok=True)
+    manifest_path = Path(args.manifest)
+    manifest = load_or_generate_manifest(
+        manifest_path, problem=args.problem, scenes=args.scenes,
+        sources=args.sources, directional=args.directional,
+        start_seed=args.start_seed, config=config)
+    scenes = scenes_from_manifest(manifest)
+    variants = [item.strip() for item in args.variants.split(",") if item.strip()]
+    if not variants:
+        parser.error("--variants 不能为空")
+    if "hrl" in variants and not args.hrl_model:
+        # manifest 中也允许写明模型路径；此处先支持 CLI 显式路径。
+        print("variant=hrl 需要 --hrl-model（P1 冒烟模型路径）。", file=sys.stderr)
+        return 2
+    known = {"hrl_heuristic", "bayes", "directional", "hrl"}
+    unknown = sorted(set(variants) - known)
+    if unknown:
+        parser.error(f"未知 variant: {unknown}")
+
+    rows: list[dict[str, Any]] = []
+    runs_root = output / "runs"
+    wall_started = time.monotonic()
+    for scene in scenes:
+        for variant in variants:
+            run_dir = runs_root / scene.scene_id / variant
+            print(json.dumps({"event": "run_start", "scene": scene.scene_id,
+                              "variant": variant}, ensure_ascii=False))
+            try:
+                result = run_scene_variant(
+                    scene, variant, config=config, hrl_model=args.hrl_model,
+                    run_dir=run_dir, hrl_device=args.hrl_device)
+                row = result["row"]
+            except Exception as exc:  # 单场异常也进入聚合，all_valid=false
+                row = _error_row(scene, variant, exc)
+            rows.append(row)
+            print(json.dumps({"event": "run_done", **{
+                key: row.get(key) for key in
+                ("scene_id", "variant", "cleared_count", "virtual_time_s",
+                 "valid", "program_run_time_s", "error")}}, ensure_ascii=False))
+    aggregate = _aggregate(rows)
+    aggregate["wall_seconds"] = round(time.monotonic() - wall_started, 3)
+    aggregate["manifest"] = str(manifest_path)
+    aggregate["output"] = str(output)
+    aggregate["variants_requested"] = variants
+    aggregate["scenes"] = len(scenes)
+    (output / "results.json").write_text(
+        json.dumps(rows, ensure_ascii=False, indent=2), encoding="utf-8")
+    (output / "manifest.json").write_text(
+        json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
+    (output / "aggregate.json").write_text(
+        json.dumps(aggregate, ensure_ascii=False, indent=2), encoding="utf-8")
+    fieldnames = list(rows[0].keys())
+    for row in rows:
+        for key in row:
+            if key not in fieldnames:
+                fieldnames.append(key)
+    with (output / "aggregate.csv").open("w", newline="", encoding="utf-8") as handle:
+        writer = csv.DictWriter(handle, fieldnames=fieldnames)
+        writer.writeheader()
+        writer.writerows(rows)
+    print(json.dumps(aggregate, ensure_ascii=False, indent=2))
+    return 0 if aggregate.get("all_valid") else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/pretrain_hrl_expert.py

@@ -0,0 +1,24 @@
+"""高层专家轨迹预训练入口（P2 待实现）。
+
+按 docs/hrl-sol-execution.md，P2 才运行 `hrl_heuristic` 收集
+(episode_id, state, candidates, mask, action)，再按 episode 划分训练/验证集做 BC。
+本文件在 P0+P1 阶段只提供命令行占位，避免把未实现的 BC 说成已交付。
+"""
+from __future__ import annotations
+
+import argparse
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--problem", type=int, choices=(3, 4), default=3)
+    parser.add_argument("--episodes", type=int, default=3000)
+    parser.add_argument("--output", default="output/hrl/bc_high")
+    parser.parse_args(argv)
+    raise NotImplementedError(
+        "P2 尚未实现：专家轨迹收集/BC 需要先通过 P1 smoke；"
+        "当前请使用 scripts/benchmark_hrl.py --variants hrl_heuristic 做启发式骨架验证。")
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/train_hrl_high.py

@@ -0,0 +1,476 @@
+"""高层 MaskablePPO 训练（P1）。
+
+架构与边界：
+- 环境 = :class:`mcm26b.hrl.hierarchical_env.HierarchicalEnv`，每一步是一个高层
+  宏动作（选择 0–119 槽位：clear/locate/station/fallback/保留）；
+- 训练/部署共用 :mod:`mcm26b.hrl.scheduler` 的候选、观测和动作掩码；
+- 低层默认使用几何选点；``--local-selector rl`` 仅作为接口预留并记录，
+  当前 HTTP 执行路径仍由高层宏动作终点决定，不把八方向推演当正式收益。
+
+P1 smoke 示例：
+    python scripts/train_hrl_high.py \
+        --problem 3 --stage 1 --timesteps 2048 --seed 320001 \
+        --envs 1 --vec-env dummy --device mps \
+        --config-profile production --model output/hrl/p1-smoke/q3.zip \
+        --eval-episodes 2 --verbose 0
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import platform
+import random
+import sys
+import time
+from pathlib import Path
+from typing import Any
+
+from mcm26b.config import ProblemConfig
+from mcm26b.hrl.curriculum import STAGES, training_config
+from mcm26b.hrl.constants import (ACTION_LAYOUT_VERSION, FEATURE_VERSION,
+                                  HIGH_OBS_DIM, N_CANDIDATES, REWARD_VERSION)
+
+
+def _stage_from_index(index: int):
+    if not 1 <= int(index) <= len(STAGES):
+        raise ValueError(f"stage 必须在 1..{len(STAGES)}")
+    return STAGES[int(index) - 1]
+
+
+def _profile_fidelity(profile: str) -> str:
+    if profile == "production":
+        return "production"
+    if profile == "coarse-pretrain":
+        return "fast"
+    raise ValueError(f"未知 config-profile: {profile}")
+
+
+def _reward_from_profile(profile: str):
+    from mcm26b.hrl.hierarchical_env import HRLReward
+
+    if profile == "time":
+        return HRLReward.time_only()
+    if profile == "prompt":
+        return HRLReward()
+    raise ValueError(f"未知 reward-profile: {profile}")
+
+
+def _make_env(problem: int, stage_index: int, seed: int,
+              monitor_path: str | None, profile: str,
+              reward_profile: str):
+    """顶层工厂：供 DummyVecEnv / SubprocVecEnv 调用（必须是可 pickle 的）。"""
+    from stable_baselines3.common.monitor import Monitor
+
+    from mcm26b.hrl.hierarchical_env import HierarchicalEnv
+
+    stage = _stage_from_index(stage_index)
+    env = HierarchicalEnv(
+        problem=problem,
+        stage=stage,
+        fidelity=_profile_fidelity(profile),
+        max_episode_steps=300,
+        real_budget_s=1800.0,
+        reward=_reward_from_profile(reward_profile),
+    )
+    return Monitor(env, filename=monitor_path)
+
+
+def _build_vec_env(args, monitor_paths: list[str]):
+    from stable_baselines3.common.vec_env import (DummyVecEnv, SubprocVecEnv)
+
+    from functools import partial
+
+    factories = [
+        partial(_make_env, args.problem, args.stage, args.seed + rank,
+                monitor_paths[rank], args.config_profile, args.reward_profile)
+        for rank in range(args.envs)
+    ]
+    if args.vec_env == "subproc":
+        # 每个 worker 独立 Monitor 文件；spawn 入口在 __main__ 保护内。
+        return SubprocVecEnv(factories, start_method="spawn")
+    return DummyVecEnv(factories)
+
+
+def _hash_file(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def _model_metadata(*, args, wall_s: float, n_envs: int,
+                    actual_timesteps: int) -> dict[str, Any]:
+    import numpy
+    import stable_baselines3
+    import sb3_contrib
+    import torch
+
+    stage = _stage_from_index(args.stage)
+    return {
+        "level": "high",
+        "problem": int(args.problem),
+        "stage": int(args.stage),
+        "stage_name": stage.name,
+        "algorithm": "maskable_ppo",
+        "feature_version": FEATURE_VERSION,
+        "action_layout_version": ACTION_LAYOUT_VERSION,
+        "reward_version": REWARD_VERSION,
+        "obs_dim": HIGH_OBS_DIM,
+        "act_dim": N_CANDIDATES,
+        "seed": int(args.seed),
+        "requested_timesteps": int(args.timesteps),
+        "timesteps_semantics": ("additional" if (args.resume or args.init)
+                                else "total"),
+        "actual_timesteps": int(actual_timesteps),
+        "n_envs": int(n_envs),
+        "vec_env": args.vec_env,
+        "config_profile": args.config_profile,
+        "fidelity": _profile_fidelity(args.config_profile),
+        "reward_profile": args.reward_profile,
+        "reward_version": REWARD_VERSION,
+        "local_selector": args.local_selector,
+        "local_model": args.local_model,
+        "device": args.device,
+        "wall_seconds": round(wall_s, 3),
+        "max_wall_s": float(args.max_wall_s),
+        "created_unix_s": time.time(),
+        "python": sys.version.split()[0],
+        "platform": platform.platform(),
+        "torch": torch.__version__,
+        "numpy": numpy.__version__,
+        "stable_baselines3": stable_baselines3.__version__,
+        "sb3_contrib": sb3_contrib.__version__,
+        "mps_available": bool(getattr(getattr(torch.backends, "mps", None),
+                                          "is_available", lambda: False)()),
+    }
+
+
+def _evaluate_env(model, *, problem: int, stage_index: int,
+                  episodes: int, seed: int, profile: str,
+                  reward_profile: str) -> dict[str, Any]:
+    from mcm26b.hrl.hierarchical_env import HierarchicalEnv
+
+    stage = _stage_from_index(stage_index)
+    env = HierarchicalEnv(problem=problem, stage=stage,
+                          fidelity=_profile_fidelity(profile),
+                          max_episode_steps=300, real_budget_s=1800.0,
+                          reward=_reward_from_profile(reward_profile))
+    wins = 0
+    rows = []
+    if episodes <= 0:
+        env.close()
+        return {"episodes": 0, "successes": 0, "success_rate": None, "rows": []}
+    for episode in range(episodes):
+        obs, info = env.reset(seed=seed + episode)
+        done = False
+        steps = 0
+        while not done:
+            action, _state = model.predict(
+                obs, action_masks=env.action_masks(), deterministic=True)
+            obs, reward, terminated, truncated, info = env.step(int(action))
+            done = bool(terminated or truncated)
+            steps += 1
+        # 课程阶段的小场景只需证明全部频道 accounted；完整阶段至少应清掉
+        # 场景最少的题面源数。这里不读取真值，只使用环境自身统计。
+        success = bool(
+            info.get("accounted", 0) >= 20
+            and info.get("cleared", 0) >= stage.sources_min
+            and not info.get("anomalies")
+        )
+        wins += int(success)
+        rows.append({
+            "episode": episode,
+            "seed": seed + episode,
+            "steps": steps,
+            "cleared": info.get("cleared", 0),
+            "accounted": info.get("accounted", 0),
+            "virtual_time_s": info.get("episode_virtual_cost_s"),
+            "stop_reason": info.get("stop_reason"),
+            "anomalies": info.get("anomalies"),
+            "success": success,
+        })
+    env.close()
+    return {
+        "episodes": len(rows),
+        "successes": wins,
+        "success_rate": wins / len(rows) if rows else 0.0,
+        "rows": rows,
+    }
+
+
+def _evaluate_manifest(model_path: Path, manifest_path: str, *,
+                       limit: int, problem: int) -> dict[str, Any]:
+    """可选：用 benchmark_hrl 的同一套真值隔离评测在固定 manifest 上跑已训模型。"""
+    scripts_dir = Path(__file__).resolve().parent
+    if str(scripts_dir) not in sys.path:
+        sys.path.insert(0, str(scripts_dir))
+    import benchmark_hrl as bench  # scripts/benchmark_hrl.py
+
+    manifest = json.loads(Path(manifest_path).read_text(encoding="utf-8"))
+    scenes = bench.scenes_from_manifest(manifest)
+    if limit > 0:
+        scenes = scenes[:limit]
+    rows = []
+    for scene in scenes:
+        run_dir = Path(model_path).parent / "eval-manifest" / scene.scene_id
+        result = bench.run_scene_variant(
+            scene, "hrl", config=ProblemConfig(),
+            hrl_model=str(model_path), run_dir=run_dir)
+        rows.append(result["row"])
+    aggregate = bench._aggregate(rows)
+    return aggregate
+
+
+def run_training(args) -> int:
+    import numpy as np
+    import torch
+    from sb3_contrib import MaskablePPO
+    from stable_baselines3.common.callbacks import BaseCallback
+    from stable_baselines3.common.utils import set_random_seed
+
+    torch.set_num_threads(1)
+    set_random_seed(args.seed)
+    random.seed(args.seed)
+    np.random.seed(args.seed)
+    if torch.cuda.is_available():
+        torch.cuda.manual_seed_all(args.seed)
+
+    model_path = Path(args.model)
+    if model_path.exists() and not (args.resume or args.force):
+        print(f"模型已存在：{model_path}；如需覆盖请加 --force，续训请加 --resume。",
+              file=sys.stderr)
+        return 2
+    model_path.parent.mkdir(parents=True, exist_ok=True)
+    run_dir = model_path.parent
+    monitor_paths = [
+        str(run_dir / f"high_curve_{model_path.stem}_env{rank}.monitor.csv")
+        for rank in range(args.envs)
+    ]
+    for path in monitor_paths:
+        Path(path).parent.mkdir(parents=True, exist_ok=True)
+
+    vec_env = _build_vec_env(args, monitor_paths)
+
+    if args.resume:
+        if not model_path.exists():
+            print(f"--resume 指定的模型不存在：{model_path}", file=sys.stderr)
+            return 2
+        model = MaskablePPO.load(str(model_path), env=vec_env,
+                                 device=args.device,
+                                 print_system_info=False)
+    elif args.init:
+        init_path = Path(args.init)
+        if not init_path.exists():
+            print(f"--init 指定的模型不存在：{init_path}", file=sys.stderr)
+            return 2
+        model = MaskablePPO.load(str(init_path), env=vec_env,
+                                 device=args.device,
+                                 print_system_info=False)
+    else:
+        model = MaskablePPO(
+            "MlpPolicy",
+            vec_env,
+            policy_kwargs={"net_arch": [128, 128]},
+            n_steps=args.rollout_steps,
+            batch_size=args.batch_size,
+            n_epochs=args.n_epochs,
+            gamma=1.0,
+            gae_lambda=0.95,
+            learning_rate=3e-4,
+            clip_range=0.2,
+            ent_coef=0.005,
+            vf_coef=0.5,
+            max_grad_norm=0.5,
+            seed=args.seed,
+            device=args.device,
+            verbose=args.verbose,
+        )
+        if args.init:
+            pass  # 已在上面处理；此分支仅保持语义清晰
+
+    class _WallClockStop(BaseCallback):
+        def __init__(self, max_wall_s: float):
+            super().__init__()
+            self.max_wall_s = max_wall_s
+            self.start = time.monotonic()
+
+        def _on_step(self) -> bool:
+            if time.monotonic() - self.start >= self.max_wall_s:
+                print(json.dumps({"event": "wall_clock_stop",
+                                  "elapsed_s": time.monotonic() - self.start}))
+                return False
+            return True
+
+    class _ProgressJSON(BaseCallback):
+        """向训练前端输出单行 JSON 进度；不影响 SB3 自身 logger。
+
+        前端首个 curve 点越早出现越好，因此开头几步按更小间隔发送（前 8 个
+        进度点逐步放大到 ``every_steps``），之后稳定在 ``every_steps``。
+        这样短程 smoke 也能看到完整曲线，长训不会刷屏。
+        """
+
+        def __init__(self, every_steps: int):
+            super().__init__()
+            self.every_steps = max(1, int(every_steps))
+            self.last_emit = int(model.num_timesteps)
+            self.emitted = 0
+
+        def _next_gap(self) -> int:
+            warmup = max(0.2, 1.0 - self.emitted / 8.0)
+            return max(1, min(self.every_steps, int(round(self.every_steps * warmup))))
+
+        def _on_step(self) -> bool:
+            current = int(self.model.num_timesteps)
+            if current - self.last_emit < self._next_gap():
+                return True
+            self.last_emit = current
+            self.emitted += 1
+            values = getattr(self.model.logger, "name_to_value", {}) or {}
+
+            def _num(key, default=None):
+                value = values.get(key)
+                if value is None:
+                    return default
+                try:
+                    value = float(value)
+                except (TypeError, ValueError):
+                    return default
+                if value != value or value in (float("inf"), float("-inf")):
+                    return default
+                return value
+
+            ep_rewards = []
+            ep_lengths = []
+            for item in getattr(self.model, "ep_info_buffer", []) or []:
+                try:
+                    ep_rewards.append(float(item.get("r")))
+                    ep_lengths.append(float(item.get("l")))
+                except (TypeError, ValueError, AttributeError):
+                    continue
+            payload = {
+                "event": "progress",
+                "timesteps": current,
+                "n_updates": int(getattr(self.model, "_n_updates", 0) or 0),
+                "ep_rew_mean": (sum(ep_rewards) / len(ep_rewards)
+                                if ep_rewards else None),
+                "ep_len_mean": (sum(ep_lengths) / len(ep_lengths)
+                                if ep_lengths else None),
+                "value_loss": _num("train/value_loss"),
+                "policy_gradient_loss": _num("train/policy_gradient_loss"),
+                "approx_kl": _num("train/approx_kl"),
+                "entropy_loss": _num("train/entropy_loss"),
+                "explained_variance": _num("train/explained_variance"),
+                "fps": _num("time/fps"),
+            }
+            print(json.dumps(payload, ensure_ascii=False, allow_nan=False),
+                  flush=True)
+            return True
+
+    progress_every = (args.progress_every if args.progress_every > 0
+                      else args.rollout_steps * args.envs)
+    callbacks = [_WallClockStop(args.max_wall_s)]
+    if args.progress_json:
+        callbacks.insert(0, _ProgressJSON(progress_every))
+
+    started = time.monotonic()
+    actual_timesteps = int(model.num_timesteps)
+    try:
+        model.learn(total_timesteps=args.timesteps,
+                    callback=callbacks if len(callbacks) > 1 else callbacks[0],
+                    progress_bar=False,
+                    reset_num_timesteps=not (args.resume or args.init))
+    except Exception as exc:  # 训练失败也必须保存失败上下文，不静默
+        failure = {
+            "event": "training_failed",
+            "error": f"{type(exc).__name__}: {exc}",
+            "model": str(model_path),
+        }
+        (run_dir / "training_failure.json").write_text(
+            json.dumps(failure, ensure_ascii=False, indent=2), encoding="utf-8")
+        raise
+    wall_s = time.monotonic() - started
+    actual_timesteps = int(model.num_timesteps)
+    model.save(str(model_path))
+    metadata = _model_metadata(args=args, wall_s=wall_s,
+                               n_envs=args.envs,
+                               actual_timesteps=actual_timesteps)
+    metadata["model_sha256"] = _hash_file(model_path)
+    metadata["monitor_files"] = monitor_paths
+    metadata["reward_note"] = (
+        "reward_profile=time 使用 HRLReward.time_only（P0.4 纯时间口径）；"
+        "reward_profile=prompt 使用原始 §5.2 塑形。模型元数据已固化口径。")
+    metadata_path = model_path.with_suffix(".json")
+    metadata_path.write_text(json.dumps(metadata, ensure_ascii=False, indent=2),
+                             encoding="utf-8")
+    vec_env.close()
+
+    evaluation = _evaluate_env(
+        model, problem=args.problem, stage_index=args.stage,
+        episodes=args.eval_episodes, seed=args.seed + 1_000_000,
+        profile=args.config_profile, reward_profile=args.reward_profile)
+    manifest_eval = None
+    if args.eval_manifest:
+        try:
+            manifest_eval = _evaluate_manifest(
+                model_path, args.eval_manifest, limit=args.eval_manifest_limit,
+                problem=args.problem)
+        except Exception as exc:
+            manifest_eval = {"error": f"{type(exc).__name__}: {exc}"}
+    metadata["evaluation"] = evaluation
+    metadata["eval_manifest"] = manifest_eval
+    metadata_path.write_text(json.dumps(metadata, ensure_ascii=False, indent=2),
+                             encoding="utf-8")
+    print(json.dumps(metadata, ensure_ascii=False, indent=2))
+    return 0
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--problem", type=int, choices=(3, 4), required=True)
+    parser.add_argument("--stage", type=int, choices=(1, 2, 3, 4),
+                        default=4)
+    parser.add_argument("--timesteps", type=int, required=True)
+    parser.add_argument("--seed", type=int, default=320001)
+    parser.add_argument("--envs", type=int, default=1)
+    parser.add_argument("--vec-env", choices=("dummy", "subproc"), default="dummy")
+    parser.add_argument("--device", choices=("cpu", "cuda", "mps", "auto"),
+                        default="auto")
+    parser.add_argument("--config-profile",
+                        choices=("production", "coarse-pretrain"),
+                        default="production")
+    parser.add_argument("--model", required=True)
+    parser.add_argument("--resume", action="store_true")
+    parser.add_argument("--init", default=None,
+                        help="可选：Q3→Q4 或继续预训练权重路径")
+    parser.add_argument("--force", action="store_true")
+    parser.add_argument("--eval-manifest", default=None)
+    parser.add_argument("--eval-manifest-limit", type=int, default=0)
+    parser.add_argument("--eval-episodes", type=int, default=5)
+    parser.add_argument("--local-selector", choices=("geometry", "rl"),
+                        default="geometry")
+    parser.add_argument("--local-model", default=None)
+    parser.add_argument("--rollout-steps", type=int, default=256)
+    parser.add_argument("--batch-size", type=int, default=256)
+    parser.add_argument("--n-epochs", type=int, default=10)
+    parser.add_argument("--max-wall-s", type=float, default=1200.0,
+                        help="训练墙钟硬上限；默认 20 分钟")
+    parser.add_argument("--reward-profile", choices=("time", "prompt"),
+                        default="time",
+                        help="time=P0.4 纯时间口径；prompt=原始 §5.2 塑形对照")
+    parser.add_argument("--progress-json", action="store_true",
+                        help="向训练前端输出单行 JSON 进度（UI 自动启用）")
+    parser.add_argument("--progress-every", type=int, default=0,
+                        help="JSON 进度间隔环境步数；0=每个 rollout")
+    parser.add_argument("--verbose", type=int, default=1)
+    args = parser.parse_args(argv)
+    if args.timesteps <= 0 or args.envs <= 0:
+        parser.error("timesteps 与 envs 必须为正")
+    if args.envs * args.rollout_steps < 2:
+        parser.error("PPO rollout 至少需要 2 个样本")
+    if args.eval_episodes < 0:
+        parser.error("eval-episodes 不能为负")
+    return run_training(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/train_hrl_low.py

@@ -0,0 +1,267 @@
+"""低层 PPO 训练：8 方向到达任务（BC 直线专家热启动 + PPO 微调）。
+
+产物：
+    output/hrl/hrl_low.zip / hrl_low.json          PPO 模型（部署用）
+    output/hrl/bc_low_expert.npz / bc_low.pt       行为克隆数据与网络
+    output/hrl/hrl_low_curve_env*.monitor.csv     每个环境独立学习曲线
+
+评估口径：到达成功率 + 路径效率（直线距离/实际路径长，1 为完美直线）。
+这是独立导航实验：目前低层 PPO 仅供轨迹诊断，不影响正式 HTTP 任务。
+
+示例：
+    uv run python scripts/train_hrl_low.py --timesteps 60000
+    uv run python scripts/train_hrl_low.py --skip-bc   # 纯 PPO 对照
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import math
+import time
+from pathlib import Path
+
+from mcm26b.config import ProblemConfig
+from mcm26b.hrl.constants import (FEATURE_VERSION, LOW_MAX_STEPS,
+                                  LOW_NUM_ACTIONS, LOW_OBS_DIM)
+from mcm26b.hrl.low_level_env import LowLevelEnv
+
+HIDDEN = (64, 64)
+
+
+def _bc_module(obs_dim: int, n_actions: int, hidden: tuple[int, ...]):
+    import torch
+
+    class BCMlp(torch.nn.Module):
+        def __init__(self):
+            super().__init__()
+            layers = []
+            prev = obs_dim
+            for width in hidden:
+                layers += [torch.nn.Linear(prev, width), torch.nn.Tanh()]
+                prev = width
+            layers.append(torch.nn.Linear(prev, n_actions))
+            self.net = torch.nn.Sequential(*layers)
+
+        def forward(self, x):
+            return self.net(x)
+
+    return BCMlp
+
+
+def collect(episodes: int, seed: int, config: ProblemConfig):
+    import numpy as np
+    env = LowLevelEnv(config=config, seed=seed)
+    observations, actions, episode_ids = [], [], []
+    for episode in range(episodes):
+        obs, _info = env.reset(seed=seed + episode)
+        for _step in range(LOW_MAX_STEPS + 2):
+            observations.append(obs)
+            actions.append(env.expert_action())
+            episode_ids.append(episode)
+            obs, _r, terminated, truncated, _info = env.step(env.expert_action())
+            if terminated or truncated:
+                break
+    env.close()
+    return (np.asarray(observations, dtype=np.float32),
+            np.asarray(actions, dtype=np.int64),
+            np.asarray(episode_ids, dtype=np.int64))
+
+
+def bc_validation_mask(episode_ids, *, seed: int):
+    """按整局留出，避免同一轨迹相邻状态跨训练/验证集。"""
+    import numpy as np
+    unique = np.unique(episode_ids)
+    if len(unique) < 2:
+        raise ValueError("BC 至少需要 2 局才能按整局划分训练/验证集")
+    rng = np.random.default_rng(seed)
+    validation_ids = rng.permutation(unique)[:max(1, len(unique) // 10)]
+    return np.isin(episode_ids, validation_ids)
+
+
+def train_bc(x, y, episode_ids, *, epochs: int, batch_size: int, lr: float,
+             device: str, seed: int):
+    import numpy as np
+    import torch
+    BCMlp = _bc_module(LOW_OBS_DIM, LOW_NUM_ACTIONS, HIDDEN)
+    model = BCMlp().to(device)
+    optimizer = torch.optim.AdamW(model.parameters(), lr=lr)
+    valid = torch.as_tensor(
+        bc_validation_mask(episode_ids, seed=seed), device=device)
+    x_t = torch.as_tensor(x, device=device)
+    y_t = torch.as_tensor(y, device=device)
+    x_v, y_v, x_tr, y_tr = x_t[valid], y_t[valid], x_t[~valid], y_t[~valid]
+    loss_fn = torch.nn.CrossEntropyLoss()
+    for epoch in range(epochs):
+        order = torch.randperm(len(x_tr), device=device)
+        total = correct = 0.0
+        for start in range(0, len(order), batch_size):
+            idx = order[start:start + batch_size]
+            logits = model(x_tr[idx])
+            loss = loss_fn(logits, y_tr[idx])
+            optimizer.zero_grad()
+            loss.backward()
+            optimizer.step()
+            total += float(loss.item()) * len(idx)
+            correct += int((logits.argmax(1) == y_tr[idx]).sum().item())
+        if epoch == 0 or (epoch + 1) % 5 == 0:
+            with torch.no_grad():
+                valid_acc = float((model(x_v).argmax(1) == y_v).float().mean().item())
+            print(json.dumps({"event": "bc_epoch", "epoch": epoch + 1,
+                              "loss": round(total / len(x_tr), 4),
+                              "train_acc": round(correct / len(x_tr), 4),
+                              "valid_acc": round(valid_acc, 4)},
+                             ensure_ascii=False))
+    with torch.no_grad():
+        valid_acc = float((model(x_v).argmax(1) == y_v).float().mean().item())
+    return model, valid_acc
+
+
+def copy_into_ppo(bc_model, device: str, env, *, seed: int,
+                  n_steps: int = 512) -> object:
+    """把 BC 权重拷入 MaskablePPO（policy_net 两层 + action_net），前向校验。"""
+    import torch
+    from sb3_contrib import MaskablePPO
+    ppo = MaskablePPO("MlpPolicy", env, policy_kwargs={"net_arch": list(HIDDEN)},
+                      n_steps=n_steps, batch_size=128, n_epochs=6,
+                      gamma=1.0, gae_lambda=0.95, learning_rate=3e-4,
+                      clip_range=0.2, ent_coef=0.005, vf_coef=0.5,
+                      max_grad_norm=0.5, seed=seed, device=device)
+    state = {key.removeprefix("net."): value
+             for key, value in bc_model.state_dict().items()}
+    policy_state = {key: value for key, value in state.items()
+                    if key in ("0.weight", "0.bias", "2.weight", "2.bias")}
+    ppo.policy.mlp_extractor.policy_net.load_state_dict(policy_state)
+    ppo.policy.action_net.load_state_dict(
+        {"weight": state["4.weight"], "bias": state["4.bias"]})
+    probe = torch.randn((8, LOW_OBS_DIM), device=device)
+    with torch.no_grad():
+        copied = ppo.policy.action_net(ppo.policy.mlp_extractor.policy_net(probe))
+        if not torch.allclose(bc_model(probe), copied, atol=1e-6):
+            raise RuntimeError("行为克隆与 PPO 前向映射不一致")
+    return ppo
+
+
+def evaluate(model, *, episodes: int, seed: int,
+             config: ProblemConfig) -> dict:
+    """确定性策略评估：成功率与路径效率（对照直线）。"""
+    env = LowLevelEnv(config=config, seed=seed)
+    successes = 0
+    efficiencies = []
+    for episode in range(episodes):
+        obs, info = env.reset(seed=seed + episode)
+        start, goal = tuple(info["position"]), tuple(info["goal"])
+        straight = math.dist(start, goal)
+        steps = 0
+        for _ in range(LOW_MAX_STEPS + 2):
+            action, _state = model.predict(obs, deterministic=True)
+            obs, _r, terminated, truncated, info = env.step(int(action))
+            steps += 1
+            if terminated or truncated:
+                break
+        successes += int(terminated)
+        efficiencies.append(min(1.0, straight /
+                                max(info["path_length_m"], 1e-9)))
+    env.close()
+    return {"episodes": episodes,
+            "success_rate": successes / episodes,
+            "mean_path_efficiency": sum(efficiencies) / len(efficiencies)}
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--timesteps", type=int, default=60_000)
+    parser.add_argument("--seed", type=int, default=20260911)
+    parser.add_argument("--bc-episodes", type=int, default=600)
+    parser.add_argument("--bc-epochs", type=int, default=12)
+    parser.add_argument("--envs", type=int, default=4)
+    parser.add_argument("--rollout-steps", type=int, default=512,
+                        help="每环境每次 PPO rollout 长度；短 smoke 可设 8")
+    parser.add_argument("--device", default="cpu")
+    parser.add_argument("--model", default="output/hrl/hrl_low.zip")
+    parser.add_argument("--eval-episodes", type=int, default=40)
+    parser.add_argument("--skip-bc", action="store_true",
+                        help="跳过行为克隆（纯 PPO 对照实验用）")
+    parser.add_argument("--verbose", type=int, default=1)
+    args = parser.parse_args(argv)
+    if min(args.envs, args.timesteps, args.eval_episodes,
+           args.rollout_steps) <= 0:
+        parser.error("envs/timesteps/eval-episodes/rollout-steps 必须为正数")
+    if args.envs * args.rollout_steps < 2:
+        parser.error("PPO rollout 至少需要 2 个样本")
+    if not args.skip_bc and (args.bc_episodes < 2 or args.bc_epochs < 1):
+        parser.error("BC 至少需要 2 局和 1 个 epoch")
+
+    import torch
+    torch.set_num_threads(1)
+    from sb3_contrib import MaskablePPO
+    from stable_baselines3.common.monitor import Monitor
+    from stable_baselines3.common.vec_env import DummyVecEnv
+    from stable_baselines3.common.utils import set_random_seed
+    set_random_seed(args.seed)
+
+    model_path = Path(args.model)
+    model_path.parent.mkdir(parents=True, exist_ok=True)
+    curve_paths = [model_path.with_name(
+        f"{model_path.stem}_curve_env{rank}.monitor.csv")
+        for rank in range(args.envs)]
+
+    def _factory(rank: int):
+        def _make():
+            env = Monitor(LowLevelEnv(seed=args.seed + rank),
+                          filename=str(curve_paths[rank]))
+            return env
+        return _make
+
+    vec_env = DummyVecEnv([_factory(rank) for rank in range(args.envs)])
+    started = time.monotonic()
+    bc_valid_acc = None
+    if args.skip_bc:
+        model = MaskablePPO("MlpPolicy", vec_env,
+                            policy_kwargs={"net_arch": list(HIDDEN)},
+                            n_steps=args.rollout_steps, batch_size=128, n_epochs=6,
+                            gamma=1.0, gae_lambda=0.95, learning_rate=3e-4,
+                            clip_range=0.2, ent_coef=0.005, vf_coef=0.5,
+                            max_grad_norm=0.5, seed=args.seed,
+                            device=args.device, verbose=args.verbose)
+    else:
+        x, y, episode_ids = collect(args.bc_episodes, args.seed, ProblemConfig())
+        bc_model, bc_valid_acc = train_bc(
+            x, y, episode_ids, epochs=args.bc_epochs, batch_size=256,
+            lr=3e-4, device=args.device, seed=args.seed)
+        model = copy_into_ppo(bc_model, args.device, vec_env, seed=args.seed,
+                              n_steps=args.rollout_steps)
+        bc_dir = model_path.parent / "bc_low.pt"
+        torch.save(bc_model.state_dict(), str(bc_dir))
+        print(json.dumps({"event": "bc_done", "samples": int(len(x))},
+                         ensure_ascii=False))
+    model.learn(total_timesteps=args.timesteps,
+                progress_bar=args.verbose > 0)
+    elapsed_s = time.monotonic() - started
+
+    model.save(str(model_path))
+    eval_seed = args.seed + 1_000_000
+    evaluation = evaluate(model, episodes=args.eval_episodes, seed=eval_seed,
+                          config=ProblemConfig())
+    report = {
+        "level": "low", "timesteps": args.timesteps, "seed": args.seed,
+        "device": args.device, "wall_seconds": round(elapsed_s, 1),
+        "model": str(model_path), "algorithm": "maskable_ppo",
+        "feature_version": FEATURE_VERSION, "bc_skipped": args.skip_bc,
+        "bc_valid_acc": bc_valid_acc, "evaluation_seed": eval_seed,
+        "monitor_files": [str(path) for path in curve_paths],
+        "training_role": "standalone_navigation_only",
+        "obs_dim": LOW_OBS_DIM, "act_dim": LOW_NUM_ACTIONS,
+        "model_sha256": hashlib.sha256(model_path.read_bytes()).hexdigest(),
+        **evaluation,
+    }
+    model_path.with_suffix(".json").write_text(
+        json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
+    print(json.dumps(report, ensure_ascii=False, indent=2))
+    vec_env.close()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/train_hrl_ui.py

@@ -0,0 +1,5 @@
+"""启动 HRL 模型训练前端（薄入口，便于加入支撑材料命令）。"""
+from mcm26b.training_ui.server import run_training_ui
+
+if __name__ == "__main__":
+    raise SystemExit(run_training_ui())

scripts/tune_hrl.py

@@ -0,0 +1,27 @@
+"""HRL Optuna TPE 调参入口（P2 待实现）。
+
+搜索空间按用户 Prompt §10，但只允许在不破坏覆盖证明/证书/预算的工程参数上搜索：
+粒子数、重采样阈值、信息增益权重、移动成本权重、检测成本权重、候选点距离、
+覆盖网格间距、停止连续无信号次数、清除距离阈值。目标 = 平均定位清除时间 +
+未清除惩罚，每个配置至少 5 个随机种子。
+"""
+from __future__ import annotations
+
+import argparse
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--problem", type=int, choices=(3, 4), default=3)
+    parser.add_argument("--trials", type=int, default=30)
+    parser.add_argument("--timesteps", type=int, default=2048)
+    parser.add_argument("--manifest", default="output/hrl/manifests/dev.json")
+    parser.add_argument("--output", default="output/hrl/p2-tuning")
+    parser.parse_args(argv)
+    raise NotImplementedError(
+        "P2 尚未实现：调参必须建立在 P1 smoke 与稳定基准之上，"
+        "且不能搜索真实误差上限或破坏覆盖证明的网格参数。")
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())