revert: 移除全部机器学习代码，回退到无 ML 版本
删除 RL/HRL/训练前端及其文档、脚本、测试与依赖：
- src/mcm26b/{rl,hrl,training_ui}/、planner/{rl,hrl}_loop.py、process_control.py
- project/（PPO 策略、训练、环境、评估）
- scripts/{train_rl,train_hrl_*,benchmark_rl,benchmark_hrl,pretrain_*_expert 等}
- tests/{test_rl*,test_hrl*,test_training_ui*}、tests/js/training_ui_harness.cjs
- docs/{algo-rl,hrl-*,training-ui*} 与相关计划条目、论文 PDF 回退
CLI 恢复为 grid/seven/bayes + directional，移除 --strategy rl|hrl、
--q4-strategy、--rl-model、--hrl-* 等参数；pyproject 删除 rl 依赖组与
PyTorch CUDA index，uv.lock 同步重生成。
保留此后的非 ML 改动（referee 回放、Q1/Q3 脚本、代码清理等）。
验证：pytest 232 passed；Q3/Q4 --self-test 全清通过。
This commit is contained in:
Horldsence
@@ -1,25 +0,0 @@
|
||||
"""HRL 消融入口（P2 待实现）。
|
||||
|
||||
预定义消融维度：时间目标 vs prompt 塑形、信息增益奖励系数、动作掩码取消范围、
|
||||
课程学习、粒子统计特征。硬性协议守卫/证书/预算永远保留，不在消融中取消。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--variants", default="time,prompt,no_info,no_mask,no_curriculum")
|
||||
parser.add_argument("--manifest", default="output/hrl/manifests/final.json")
|
||||
parser.add_argument("--output", default="output/hrl/p2-ablation")
|
||||
parser.add_argument("--timesteps", type=int, default=2048)
|
||||
parser.add_argument("--seeds", type=int, default=5)
|
||||
parser.parse_args(argv)
|
||||
raise NotImplementedError(
|
||||
"P2 尚未实现：必须先完成专家 BC 与冻结场景对照，"
|
||||
"再对相同训练步数/场景/种子逐项消融；P1 只保证 benchmark/training/CLI。")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,428 +0,0 @@
|
||||
"""HRL/基线统一评测入口（P1）。
|
||||
|
||||
特点：
|
||||
- 同一 manifest 中的真值场景被所有 variant 复用，保证配对比较；
|
||||
- 真值仅在测试端用于构造 ``LocalSimulator`` 与最终核对，策略端只收到协议响应；
|
||||
- 支持 ``hrl_heuristic``（高层启发式降级）、``bayes``、``directional``；
|
||||
- 支持已训练高层模型 ``hrl``（需 ``--hrl-model`` 或 manifest 中的 variant 配置）。
|
||||
|
||||
输出目录（默认拒绝覆盖已有结果）：
|
||||
<output>/manifest.json
|
||||
<output>/runs/<scene_id>/<variant>/summary.json
|
||||
<output>/runs/<scene_id>/<variant>/session.jsonl
|
||||
<output>/results.json
|
||||
<output>/aggregate.json
|
||||
<output>/aggregate.csv
|
||||
|
||||
示例：
|
||||
python scripts/benchmark_hrl.py \
|
||||
--variants hrl_heuristic,bayes,directional \
|
||||
--manifest output/hrl/manifests/dev.json \
|
||||
--output output/hrl/p1-heuristic
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import statistics
|
||||
import sys
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from mcm26b.config import ProblemConfig
|
||||
from mcm26b.harness import LocalSimTransport, mixed_scene, random_omni_scene
|
||||
from mcm26b.harness.scenes import referee_spec
|
||||
from mcm26b.protocol import SimulatorClient
|
||||
from mcm26b.protocol.log import SessionLog
|
||||
|
||||
try: # 测试端依赖；运行时代码不导入
|
||||
from referee import LocalSimulator, Source
|
||||
except ImportError as exc: # pragma: no cover
|
||||
raise SystemExit("benchmark_hrl 需要 dev 依赖 referee。请先 `uv sync`。") from exc
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# manifest / scene
|
||||
# ---------------------------------------------------------------------------
|
||||
@dataclass(frozen=True)
|
||||
class SceneSpec:
|
||||
scene_id: str
|
||||
seed: int
|
||||
error_seed: int
|
||||
problem: int
|
||||
sources: list[dict[str, Any]]
|
||||
|
||||
@property
|
||||
def truth_channels(self) -> tuple[int, ...]:
|
||||
return tuple(sorted(int(src["channel"]) for src in self.sources))
|
||||
|
||||
|
||||
def _source_dict(source: Source) -> dict[str, Any]:
|
||||
return {
|
||||
"channel": int(source.channel),
|
||||
"x": float(source.x),
|
||||
"y": float(source.y),
|
||||
"r_eff_m": float(source.r_eff_m),
|
||||
"directional": bool(source.directional),
|
||||
"heading_deg": float(source.heading_deg),
|
||||
}
|
||||
|
||||
|
||||
def _sources_to_dicts(sources: list[Source]) -> list[dict[str, Any]]:
|
||||
return [_source_dict(source) for source in sources]
|
||||
|
||||
|
||||
def _source_from_dict(data: dict[str, Any]) -> Source:
|
||||
return Source(
|
||||
channel=int(data["channel"]),
|
||||
x=float(data["x"]),
|
||||
y=float(data["y"]),
|
||||
r_eff_m=float(data["r_eff_m"]),
|
||||
directional=bool(data.get("directional", False)),
|
||||
heading_deg=float(data.get("heading_deg", 0.0)),
|
||||
)
|
||||
|
||||
|
||||
def make_default_manifest(*, problem: int, scenes: int, sources: int | None,
|
||||
directional: int, start_seed: int,
|
||||
config: ProblemConfig) -> dict[str, Any]:
|
||||
"""按 CLI 参数生成可复现的固定场景清单。"""
|
||||
entries: list[dict[str, Any]] = []
|
||||
for index in range(scenes):
|
||||
seed = start_seed + index
|
||||
if problem == 4:
|
||||
scene_sources = mixed_scene(seed=seed, count=sources,
|
||||
directional=directional, config=config)
|
||||
else:
|
||||
scene_sources = random_omni_scene(seed=seed, count=sources,
|
||||
config=config)
|
||||
entries.append({
|
||||
"scene_id": f"scene-{seed}",
|
||||
"seed": seed,
|
||||
"error_seed": seed + 10_000,
|
||||
"problem": problem,
|
||||
"sources": _sources_to_dicts(scene_sources),
|
||||
})
|
||||
return {"version": 1, "problem": problem, "scenes": entries}
|
||||
|
||||
|
||||
def load_or_generate_manifest(path: Path, *, problem: int, scenes: int,
|
||||
sources: int | None, directional: int,
|
||||
start_seed: int,
|
||||
config: ProblemConfig) -> dict[str, Any]:
|
||||
if path.exists():
|
||||
manifest = json.loads(path.read_text(encoding="utf-8"))
|
||||
else:
|
||||
manifest = make_default_manifest(
|
||||
problem=problem, scenes=scenes, sources=sources,
|
||||
directional=directional, start_seed=start_seed, config=config)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8")
|
||||
if "scenes" not in manifest or not manifest["scenes"]:
|
||||
raise ValueError(f"manifest 无 scenes: {path}")
|
||||
for raw in manifest["scenes"]:
|
||||
if not {"scene_id", "error_seed", "problem", "sources"} <= set(raw):
|
||||
raise ValueError(f"manifest scene 字段缺失: {raw!r}")
|
||||
if not raw["sources"]:
|
||||
raise ValueError(f"scene {raw.get('scene_id')} 没有来源真值")
|
||||
return manifest
|
||||
|
||||
|
||||
def scenes_from_manifest(manifest: dict[str, Any]) -> list[SceneSpec]:
|
||||
result: list[SceneSpec] = []
|
||||
for raw in manifest["scenes"]:
|
||||
result.append(SceneSpec(
|
||||
scene_id=str(raw["scene_id"]),
|
||||
seed=int(raw.get("seed", 0)),
|
||||
error_seed=int(raw["error_seed"]),
|
||||
problem=int(raw["problem"]),
|
||||
sources=[dict(item) for item in raw["sources"]],
|
||||
))
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# variants
|
||||
# ---------------------------------------------------------------------------
|
||||
def _build_loop(variant: str, client: SimulatorClient, config: ProblemConfig,
|
||||
problem: int, *, hrl_model: str | None,
|
||||
hrl_device: str = "auto"):
|
||||
"""延迟导入变体实现，避免 benchmark 顶层拉起重依赖。"""
|
||||
if variant == "bayes":
|
||||
from mcm26b.planner.bayes_loop import BayesLoop
|
||||
return BayesLoop(client, config)
|
||||
if variant == "directional":
|
||||
from mcm26b.planner import DirectionalExtension
|
||||
return DirectionalExtension(client, config)
|
||||
if variant == "hrl_heuristic":
|
||||
from mcm26b.planner.hrl_loop import HRLLoop
|
||||
return HRLLoop(client, config, problem=problem, high_policy=None)
|
||||
if variant == "hrl":
|
||||
if not hrl_model:
|
||||
raise RuntimeError("variant=hrl 需要 --hrl-model")
|
||||
from mcm26b.hrl.policy import HighLevelPolicy
|
||||
from mcm26b.planner.hrl_loop import HRLLoop
|
||||
policy = HighLevelPolicy(hrl_model, device=hrl_device, problem=problem)
|
||||
return HRLLoop(client, config, problem=problem, high_policy=policy)
|
||||
raise ValueError(f"未知 variant: {variant!r}")
|
||||
|
||||
|
||||
def _validity(summary: dict[str, Any], truth_channels: tuple[int, ...]) -> dict[str, Any]:
|
||||
"""测试端有效性审计。真值只在此处进入。"""
|
||||
cleared = set(int(c) for c in summary.get("cleared_channels", []))
|
||||
absent = set(int(c) for c in summary.get("absent_channels", []))
|
||||
truth = set(truth_channels)
|
||||
declared_absent = truth & absent
|
||||
cleared_matches_truth = cleared == truth
|
||||
no_source_declared_absent = not declared_absent
|
||||
completed = bool(summary.get("completed"))
|
||||
anomalies = list(summary.get("anomalies", []))
|
||||
# 本地 benchmark 要求：清洗真值且无守恒性异常；缺席集合不得含真值。
|
||||
valid = completed and cleared_matches_truth and no_source_declared_absent \
|
||||
and not anomalies
|
||||
return {
|
||||
"valid": valid,
|
||||
"completed": completed,
|
||||
"cleared_matches_truth": cleared_matches_truth,
|
||||
"no_source_declared_absent": no_source_declared_absent,
|
||||
"truth_channels": sorted(truth),
|
||||
"declared_absent_truth": sorted(declared_absent),
|
||||
"anomaly_count": len(anomalies),
|
||||
}
|
||||
|
||||
|
||||
def _row_from_summary(*, scene: SceneSpec, variant: str, summary: dict[str, Any],
|
||||
wall_s: float) -> dict[str, Any]:
|
||||
validity = _validity(summary, scene.truth_channels)
|
||||
cleared_count = len(summary.get("cleared_channels", []))
|
||||
virtual_s = float(summary.get("virtual_time_s") or 0.0)
|
||||
avg_s = summary.get("avg_locate_clear_time_s")
|
||||
totals = summary.get("step_totals", {}) or {}
|
||||
return {
|
||||
"scene_id": scene.scene_id,
|
||||
"variant": variant,
|
||||
"problem": scene.problem,
|
||||
"truth_count": len(scene.truth_channels),
|
||||
"cleared_count": cleared_count,
|
||||
"virtual_time_s": virtual_s,
|
||||
"avg_locate_clear_time_s": (float(avg_s) if avg_s is not None else None),
|
||||
"accounted_virtual_time_s": summary.get("accounted_virtual_time_s"),
|
||||
"program_run_time_s": round(wall_s, 6),
|
||||
"real_elapsed_total_s": totals.get("real_elapsed_total_s"),
|
||||
"total_requests": summary.get("total_requests"),
|
||||
"measures": summary.get("measures"),
|
||||
"clears": summary.get("clears"),
|
||||
"clear_success": summary.get("clear_success"),
|
||||
"completed": bool(summary.get("completed")),
|
||||
"stop_reason": summary.get("stop_reason"),
|
||||
"exit_reason": summary.get("exit_reason"),
|
||||
**validity,
|
||||
}
|
||||
|
||||
|
||||
def _error_row(scene: SceneSpec, variant: str, exc: BaseException) -> dict[str, Any]:
|
||||
return {
|
||||
"scene_id": scene.scene_id,
|
||||
"variant": variant,
|
||||
"problem": scene.problem,
|
||||
"truth_count": len(scene.truth_channels),
|
||||
"cleared_count": 0,
|
||||
"virtual_time_s": None,
|
||||
"avg_locate_clear_time_s": None,
|
||||
"accounted_virtual_time_s": None,
|
||||
"program_run_time_s": None,
|
||||
"real_elapsed_total_s": None,
|
||||
"total_requests": None,
|
||||
"measures": None,
|
||||
"clears": None,
|
||||
"clear_success": None,
|
||||
"completed": False,
|
||||
"stop_reason": None,
|
||||
"exit_reason": None,
|
||||
"valid": False,
|
||||
"cleared_matches_truth": False,
|
||||
"no_source_declared_absent": False,
|
||||
"truth_channels": sorted(scene.truth_channels),
|
||||
"declared_absent_truth": [],
|
||||
"anomaly_count": 0,
|
||||
"error": f"{type(exc).__name__}: {exc}",
|
||||
}
|
||||
|
||||
|
||||
def run_scene_variant(scene: SceneSpec, variant: str, *, config: ProblemConfig,
|
||||
hrl_model: str | None, run_dir: Path,
|
||||
hrl_device: str = "auto") -> dict[str, Any]:
|
||||
"""运行一个 scene × variant。返回可序列化结果行与 summary。"""
|
||||
run_dir.mkdir(parents=True, exist_ok=True)
|
||||
sources = [_source_from_dict(raw) for raw in scene.sources]
|
||||
simulator = LocalSimulator(
|
||||
sources,
|
||||
robot_id="hrl-benchmark",
|
||||
spec=referee_spec(config),
|
||||
error_seed=scene.error_seed,
|
||||
stress_mode=False,
|
||||
)
|
||||
# 真值只用于 LocalSimulator；策略端仅持有 client/loop，不读取 simulator.sources。
|
||||
log_path = run_dir / "session.jsonl"
|
||||
log = SessionLog(path=log_path)
|
||||
client = SimulatorClient("hrl-benchmark", config=config,
|
||||
transport=LocalSimTransport(simulator), log=log)
|
||||
loop = _build_loop(variant, client, config, scene.problem,
|
||||
hrl_model=hrl_model, hrl_device=hrl_device)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
summary = loop.run()
|
||||
finally:
|
||||
wall_s = time.monotonic() - started
|
||||
row = _row_from_summary(scene=scene, variant=variant, summary=summary,
|
||||
wall_s=wall_s)
|
||||
# 完整 summary 可能很大；仍写入 runs 目录，供复算与异常审计。
|
||||
(run_dir / "summary.json").write_text(
|
||||
json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return {"row": row, "summary": summary}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# aggregate
|
||||
# ---------------------------------------------------------------------------
|
||||
def _aggregate(rows: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for row in rows:
|
||||
grouped[row["variant"]].append(row)
|
||||
result: dict[str, Any] = {"variants": {}, "all_valid": True}
|
||||
for variant, items in sorted(grouped.items()):
|
||||
virtual = [float(item["virtual_time_s"]) for item in items
|
||||
if item.get("virtual_time_s") is not None]
|
||||
per_source = [float(item["avg_locate_clear_time_s"])
|
||||
for item in items
|
||||
if item.get("avg_locate_clear_time_s") is not None]
|
||||
cleared = [int(item["cleared_count"] or 0) for item in items]
|
||||
program = [float(item["program_run_time_s"]) for item in items
|
||||
if item.get("program_run_time_s") is not None]
|
||||
success = [1.0 if item.get("valid") else 0.0 for item in items]
|
||||
result["variants"][variant] = {
|
||||
"episodes": len(items),
|
||||
"valid_episodes": int(sum(success)),
|
||||
"valid_rate": (sum(success) / len(items)) if items else 0.0,
|
||||
"success_rate": (sum(success) / len(items)) if items else 0.0,
|
||||
"cleared_count_mean": statistics.fmean(cleared) if cleared else 0.0,
|
||||
"cleared_count_std": (statistics.pstdev(cleared)
|
||||
if len(cleared) > 1 else 0.0),
|
||||
"virtual_time_mean_s": statistics.fmean(virtual) if virtual else 0.0,
|
||||
"virtual_time_std_s": (statistics.pstdev(virtual)
|
||||
if len(virtual) > 1 else 0.0),
|
||||
"virtual_time_min_s": min(virtual) if virtual else 0.0,
|
||||
"virtual_time_max_s": max(virtual) if virtual else 0.0,
|
||||
"avg_locate_clear_time_mean_s": (statistics.fmean(per_source)
|
||||
if per_source else None),
|
||||
"program_run_time_mean_s": (statistics.fmean(program)
|
||||
if program else None),
|
||||
"total_requests_mean": statistics.fmean(
|
||||
[int(item.get("total_requests") or 0) for item in items]),
|
||||
}
|
||||
if result["variants"][variant]["valid_rate"] < 1.0:
|
||||
result["all_valid"] = False
|
||||
return result
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--variants", default="hrl_heuristic,bayes,directional",
|
||||
help="逗号分隔：hrl_heuristic,bayes,directional,hrl")
|
||||
parser.add_argument("--manifest", default="output/hrl/manifests/dev.json")
|
||||
parser.add_argument("--output", default="output/hrl/p1-benchmark")
|
||||
parser.add_argument("--problem", type=int, choices=(3, 4), default=3)
|
||||
parser.add_argument("--scenes", type=int, default=6,
|
||||
help="manifest 不存在时生成的场景数")
|
||||
parser.add_argument("--sources", type=int, default=None,
|
||||
help="每局源数；默认按题面 10~16 随机")
|
||||
parser.add_argument("--directional", type=int, default=3,
|
||||
help="问题4定向源数量（生成 manifest 时使用）")
|
||||
parser.add_argument("--start-seed", type=int, default=91_000)
|
||||
parser.add_argument("--hrl-model", default=None)
|
||||
parser.add_argument("--hrl-device", choices=("auto", "cpu", "cuda", "mps"),
|
||||
default="auto")
|
||||
parser.add_argument("--force", action="store_true",
|
||||
help="允许覆盖已有输出目录")
|
||||
parser.add_argument("--log", action="store_true",
|
||||
help="为兼容 CLI 保留；当前始终写 session.jsonl")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
config = ProblemConfig()
|
||||
output = Path(args.output)
|
||||
if output.exists() and any(output.iterdir()) and not args.force:
|
||||
print(f"输出目录已存在且非空：{output}；如需覆盖请加 --force。",
|
||||
file=sys.stderr)
|
||||
return 2
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
manifest_path = Path(args.manifest)
|
||||
manifest = load_or_generate_manifest(
|
||||
manifest_path, problem=args.problem, scenes=args.scenes,
|
||||
sources=args.sources, directional=args.directional,
|
||||
start_seed=args.start_seed, config=config)
|
||||
scenes = scenes_from_manifest(manifest)
|
||||
variants = [item.strip() for item in args.variants.split(",") if item.strip()]
|
||||
if not variants:
|
||||
parser.error("--variants 不能为空")
|
||||
if "hrl" in variants and not args.hrl_model:
|
||||
# manifest 中也允许写明模型路径；此处先支持 CLI 显式路径。
|
||||
print("variant=hrl 需要 --hrl-model（P1 冒烟模型路径）。", file=sys.stderr)
|
||||
return 2
|
||||
known = {"hrl_heuristic", "bayes", "directional", "hrl"}
|
||||
unknown = sorted(set(variants) - known)
|
||||
if unknown:
|
||||
parser.error(f"未知 variant: {unknown}")
|
||||
|
||||
rows: list[dict[str, Any]] = []
|
||||
runs_root = output / "runs"
|
||||
wall_started = time.monotonic()
|
||||
for scene in scenes:
|
||||
for variant in variants:
|
||||
run_dir = runs_root / scene.scene_id / variant
|
||||
print(json.dumps({"event": "run_start", "scene": scene.scene_id,
|
||||
"variant": variant}, ensure_ascii=False))
|
||||
try:
|
||||
result = run_scene_variant(
|
||||
scene, variant, config=config, hrl_model=args.hrl_model,
|
||||
run_dir=run_dir, hrl_device=args.hrl_device)
|
||||
row = result["row"]
|
||||
except Exception as exc: # 单场异常也进入聚合，all_valid=false
|
||||
row = _error_row(scene, variant, exc)
|
||||
rows.append(row)
|
||||
print(json.dumps({"event": "run_done", **{
|
||||
key: row.get(key) for key in
|
||||
("scene_id", "variant", "cleared_count", "virtual_time_s",
|
||||
"valid", "program_run_time_s", "error")}}, ensure_ascii=False))
|
||||
aggregate = _aggregate(rows)
|
||||
aggregate["wall_seconds"] = round(time.monotonic() - wall_started, 3)
|
||||
aggregate["manifest"] = str(manifest_path)
|
||||
aggregate["output"] = str(output)
|
||||
aggregate["variants_requested"] = variants
|
||||
aggregate["scenes"] = len(scenes)
|
||||
(output / "results.json").write_text(
|
||||
json.dumps(rows, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
(output / "manifest.json").write_text(
|
||||
json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
(output / "aggregate.json").write_text(
|
||||
json.dumps(aggregate, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
fieldnames = list(rows[0].keys())
|
||||
for row in rows:
|
||||
for key in row:
|
||||
if key not in fieldnames:
|
||||
fieldnames.append(key)
|
||||
with (output / "aggregate.csv").open("w", newline="", encoding="utf-8") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
print(json.dumps(aggregate, ensure_ascii=False, indent=2))
|
||||
return 0 if aggregate.get("all_valid") else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,128 +0,0 @@
|
||||
"""问题3/4 RL 策略与既有策略的配对自建消融。
|
||||
|
||||
同一种子、同一场景分别跑基线（Q3 bayes / Q4 directional）与 RL 变体，
|
||||
比较全部清除成功、虚拟总时间、检测次数与请求总数。结果写入
|
||||
output/rl/benchmark.json。
|
||||
|
||||
用法：
|
||||
uv sync --group rl --group dev
|
||||
uv run python scripts/benchmark_rl.py --seeds 5
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import statistics
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from mcm26b.config import ProblemConfig
|
||||
from mcm26b.evaluation import recompute_virtual_time
|
||||
from mcm26b.harness import (LocalSimTransport, mixed_scene, random_omni_scene,
|
||||
referee_spec)
|
||||
from mcm26b.planner import DirectionalExtension
|
||||
from mcm26b.planner.bayes_loop import BayesLoop
|
||||
from mcm26b.planner.rl_loop import RLBayesLoop, RLDirectionalLoop
|
||||
from mcm26b.protocol.client import SimulatorClient
|
||||
from referee import LocalSimulator
|
||||
|
||||
|
||||
def run_one(loop_cls, sources, *, config, robot_id: str, **kwargs) -> dict:
|
||||
simulator = LocalSimulator(sources, robot_id=robot_id,
|
||||
spec=referee_spec(config, max_real_duration_s=3600.0),
|
||||
stress_mode=True)
|
||||
client = SimulatorClient(robot_id, config=config,
|
||||
transport=LocalSimTransport(simulator))
|
||||
started = time.monotonic()
|
||||
summary = loop_cls(client, config, **kwargs).run() \
|
||||
if kwargs else loop_cls(client, config).run()
|
||||
events = client.log.action_events()
|
||||
successful = {e['request']['channel'] for e in events
|
||||
if e['response'].get('clear_result') == 'success'}
|
||||
truth = {s.channel for s in sources}
|
||||
return {
|
||||
"completed": bool(summary["completed"]),
|
||||
"all_cleared": successful == truth,
|
||||
"correct_absence": not bool(set(summary['absent_channels']) & truth),
|
||||
"anomalies": summary['anomalies'],
|
||||
"timing_mismatches": len(recompute_virtual_time(events)['mismatches']),
|
||||
"normal_exit": summary['exit_reason'] == 'user_exit',
|
||||
"virtual_time_s": float(summary["virtual_time_s"]),
|
||||
"measures": int(summary["measures"]),
|
||||
"total_requests": int(summary["total_requests"]),
|
||||
"move_distance_m": float(summary["move_distance_m"]),
|
||||
"wall_s": round(time.monotonic() - started, 3),
|
||||
}
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--seeds", type=int, default=5)
|
||||
parser.add_argument("--sources", type=int, default=10)
|
||||
parser.add_argument("--directional", type=int, default=3)
|
||||
parser.add_argument("--output", default="output/rl/benchmark.json")
|
||||
args = parser.parse_args(argv)
|
||||
config = ProblemConfig()
|
||||
robot_id = "2026001"
|
||||
report = {"config": {"seeds": args.seeds, "sources": args.sources,
|
||||
"directional": args.directional},
|
||||
"q3": [], "q4": []}
|
||||
|
||||
for seed in range(20260910, 20260910 + args.seeds):
|
||||
omni = random_omni_scene(seed=seed, count=args.sources, config=config)
|
||||
mixed = mixed_scene(seed=seed, count=args.sources,
|
||||
directional=args.directional, config=config)
|
||||
q3_base = run_one(BayesLoop, omni, config=config, robot_id=robot_id)
|
||||
q3_rl = run_one(RLBayesLoop, omni, config=config, robot_id=robot_id)
|
||||
q4_base = run_one(DirectionalExtension, mixed, config=config,
|
||||
robot_id=robot_id)
|
||||
q4_rl = run_one(RLDirectionalLoop, mixed, config=config,
|
||||
robot_id=robot_id)
|
||||
report["q3"].append({"seed": seed, "bayes": q3_base, "rl": q3_rl})
|
||||
report["q4"].append({"seed": seed, "directional": q4_base, "rl": q4_rl})
|
||||
print(json.dumps({"event": "seed_done", "seed": seed,
|
||||
"q3_base_s": q3_base["virtual_time_s"],
|
||||
"q3_rl_s": q3_rl["virtual_time_s"],
|
||||
"q4_base_s": q4_base["virtual_time_s"],
|
||||
"q4_rl_s": q4_rl["virtual_time_s"]},
|
||||
ensure_ascii=False))
|
||||
|
||||
for key, baseline in (("q3", "bayes"), ("q4", "directional")):
|
||||
rows = report[key]
|
||||
deltas = [row["rl"]["virtual_time_s"] - row[baseline]["virtual_time_s"]
|
||||
for row in rows]
|
||||
cleared = all(row["rl"]["all_cleared"] and row[baseline]["all_cleared"]
|
||||
for row in rows)
|
||||
valid = all(part['completed'] and part['all_cleared'] and part['correct_absence']
|
||||
and part['normal_exit'] and not part['anomalies']
|
||||
and not part['timing_mismatches']
|
||||
for row in rows for part in (row[baseline],row['rl']))
|
||||
report[key + "_summary"] = {
|
||||
"mean_virtual_s": {
|
||||
baseline: statistics.mean(row[baseline]["virtual_time_s"] for row in rows),
|
||||
"rl": statistics.mean(row["rl"]["virtual_time_s"] for row in rows),
|
||||
},
|
||||
"mean_delta_s": statistics.mean(deltas),
|
||||
"relative_delta_pct": 100.0 * statistics.mean(deltas)
|
||||
/ statistics.mean(row[baseline]["virtual_time_s"] for row in rows),
|
||||
"mean_measures": {
|
||||
baseline: statistics.mean(row[baseline]["measures"] for row in rows),
|
||||
"rl": statistics.mean(row["rl"]["measures"] for row in rows),
|
||||
},
|
||||
"all_cleared": cleared,
|
||||
"all_valid": valid,
|
||||
}
|
||||
output = Path(args.output)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(json.dumps(report, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8")
|
||||
print(json.dumps({"event": "benchmark_done", "path": str(output),
|
||||
"q3": report["q3_summary"],
|
||||
"q4": report["q4_summary"]},
|
||||
ensure_ascii=False, indent=2))
|
||||
return 0 if report['q3_summary']['all_valid'] and report['q4_summary']['all_valid'] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,125 +0,0 @@
|
||||
"""同场景配对比较手写策略、旧PPO及修订模型；真值仅在评测端使用。"""
|
||||
import argparse
|
||||
from dataclasses import asdict
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
from statistics import mean, median
|
||||
from time import perf_counter
|
||||
|
||||
import torch
|
||||
|
||||
from mcm26b.config import ProblemConfig
|
||||
from mcm26b.evaluation import recompute_virtual_time
|
||||
from mcm26b.harness import LocalSimTransport, mixed_scene, random_omni_scene, referee_spec
|
||||
from mcm26b.planner import DirectionalExtension
|
||||
from mcm26b.planner.bayes_loop import BayesLoop
|
||||
from mcm26b.planner.rl_loop import RLBayesLoop, RLDirectionalLoop
|
||||
from mcm26b.protocol.client import SimulatorClient
|
||||
from referee import LocalSimulator, Source
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--split', choices=('development','validation','stress'),
|
||||
default='development')
|
||||
parser.add_argument('--variants', default='baseline,legacy,revised')
|
||||
parser.add_argument('--problems', default='3,4')
|
||||
parser.add_argument('--model-dir', type=Path, default=Path('output/rl/review-20260911'))
|
||||
parser.add_argument('--label', required=True)
|
||||
args = parser.parse_args()
|
||||
torch.set_num_threads(1)
|
||||
config = ProblemConfig()
|
||||
output = args.model_dir / 'evaluation' / args.split / args.label
|
||||
output.mkdir(parents=True, exist_ok=False)
|
||||
variants = args.variants.split(',')
|
||||
rows = []
|
||||
def save():
|
||||
aggregates = {}
|
||||
for problem in (3,4):
|
||||
for variant in variants:
|
||||
part = [r for r in rows if r['problem']==problem and r['variant']==variant]
|
||||
if part:
|
||||
aggregates[f'q{problem}-{variant}'] = dict(
|
||||
runs=len(part), valid=sum(r['valid'] for r in part),
|
||||
mean_virtual_s=mean(r['virtual_s'] for r in part),
|
||||
median_virtual_s=median(r['virtual_s'] for r in part),
|
||||
max_virtual_s=max(r['virtual_s'] for r in part),
|
||||
failed_clears=sum(r['failed_clears'] for r in part),
|
||||
repeated_measures=sum(r['repeated_measures'] for r in part))
|
||||
result = dict(source='local_referee_only', config=asdict(config), split=args.split,
|
||||
rows=rows, aggregates=aggregates)
|
||||
(output/'results.json').write_text(
|
||||
json.dumps(result,ensure_ascii=False,indent=2),encoding='utf-8')
|
||||
return aggregates
|
||||
for problem in map(int, args.problems.split(',')):
|
||||
scenes = []
|
||||
if args.split == 'stress':
|
||||
for count in (10,16):
|
||||
scenes.append((f'boundary-{count}', count, 99000+count, [
|
||||
Source(c+1,1800*math.cos(2*math.pi*c/count),1800*math.sin(2*math.pi*c/count),
|
||||
1000,directional=(problem==4 and c%2==0),
|
||||
heading_deg=360*c/count if problem==4 and c%2==0 else 0)
|
||||
for c in range(count)]))
|
||||
else:
|
||||
seeds = range(700,703) if args.split=='development' else range(91000,91006)
|
||||
counts = (10,16) if args.split=='development' else (10,13,16)
|
||||
for count in counts:
|
||||
for seed in seeds:
|
||||
directional = (1, count//2, count-1)[seed%3]
|
||||
sources = (random_omni_scene(seed, count, config) if problem==3 else
|
||||
mixed_scene(seed, count, directional, config))
|
||||
scenes.append((f'{count}-{seed}',count,seed,sources))
|
||||
for name,count,seed,sources in scenes:
|
||||
scene_data = dict(sources=[asdict(s) for s in sources], error_seed=seed+2000000)
|
||||
scene_path = output/f'q{problem}-{name}-scene.json'
|
||||
scene_path.write_text(json.dumps(scene_data),encoding='utf-8')
|
||||
for variant in variants:
|
||||
model = None
|
||||
cls = BayesLoop if problem==3 else DirectionalExtension
|
||||
kwargs = {}
|
||||
if variant != 'baseline':
|
||||
cls = RLBayesLoop if problem==3 else RLDirectionalLoop
|
||||
model = (Path(f'output/rl/q{problem}_ppo.zip') if variant=='legacy'
|
||||
else args.model_dir/f'q{problem}_masked.zip')
|
||||
kwargs = dict(model_path=model, device='cpu')
|
||||
sim = LocalSimulator(sources,robot_id='RL-AUDIT',spec=referee_spec(config),
|
||||
error_seed=seed+2000000)
|
||||
client = SimulatorClient('RL-AUDIT',config=config,transport=LocalSimTransport(sim))
|
||||
start = perf_counter()
|
||||
summary = cls(client,config,**kwargs).run()
|
||||
events = client.log.action_events()
|
||||
truth = {s.channel for s in sources}
|
||||
successful = {e['request']['channel'] for e in events
|
||||
if e['response'].get('clear_result')=='success'}
|
||||
seen = set()
|
||||
repeats = 0
|
||||
for e in events:
|
||||
if e['path']=='/measure':
|
||||
p=e['request']['position']
|
||||
tag=(e['request']['channel'],round(p['x'],6),round(p['y'],6))
|
||||
repeats += tag in seen
|
||||
seen.add(tag)
|
||||
mismatches = recompute_virtual_time(events)['mismatches']
|
||||
valid = (successful==truth and summary['completed'] and not summary['anomalies']
|
||||
and not set(summary['absent_channels']) & truth
|
||||
and summary['exit_reason']=='user_exit' and not mismatches)
|
||||
row = dict(case=name,problem=problem,variant=variant,valid=valid,
|
||||
virtual_s=client.state.virtual_time_s,all_cleared=successful==truth,
|
||||
measures=summary['measures'],
|
||||
failed_clears=summary['clears']-len(successful),
|
||||
repeated_measures=repeats,wall_s=perf_counter()-start,
|
||||
model_sha256=(hashlib.sha256(model.read_bytes()).hexdigest()
|
||||
if model else None))
|
||||
rows.append(row)
|
||||
(output/f'q{problem}-{name}-{variant}.json').write_text(json.dumps(
|
||||
dict(row=row,summary=summary,events=events,timing_mismatches=mismatches),
|
||||
ensure_ascii=False),encoding='utf-8')
|
||||
save()
|
||||
print(json.dumps(row),flush=True)
|
||||
print(json.dumps(save(),indent=2),flush=True)
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
@@ -1,24 +0,0 @@
|
||||
"""高层专家轨迹预训练入口（P2 待实现）。
|
||||
|
||||
按 docs/hrl-sol-execution.md，P2 才运行 `hrl_heuristic` 收集
|
||||
(episode_id, state, candidates, mask, action)，再按 episode 划分训练/验证集做 BC。
|
||||
本文件在 P0+P1 阶段只提供命令行占位，避免把未实现的 BC 说成已交付。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--problem", type=int, choices=(3, 4), default=3)
|
||||
parser.add_argument("--episodes", type=int, default=3000)
|
||||
parser.add_argument("--output", default="output/hrl/bc_high")
|
||||
parser.parse_args(argv)
|
||||
raise NotImplementedError(
|
||||
"P2 尚未实现：专家轨迹收集/BC 需要先通过 P1 smoke；"
|
||||
"当前请使用 scripts/benchmark_hrl.py --variants hrl_heuristic 做启发式骨架验证。")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,255 +0,0 @@
|
||||
"""用自建模拟器的既有专家逻辑做行为克隆，初始化 PPO 策略网络。
|
||||
|
||||
思路：自建 referee 的观测模型与真值逻辑既快又可复现。先在它上面用
|
||||
手写专家（Q3: worst-posterior 选点器；Q4: 遮挡梯子选点器）生成
|
||||
(观测, 专家动作) 数据集，训练一个与 PPO policy_net 同构的 MLP 分类器，
|
||||
再把权重拷入 PPO 模型作为初始化；随后 scripts/train_rl.py --init 继续
|
||||
PPO 微调。这样 RL 从强先验出发，而不是从随机策略开始。
|
||||
|
||||
产物：
|
||||
output/rl/bc/q3_expert.npz output/rl/bc/q3_bc.pt output/rl/q3_ppo_bc_init.zip
|
||||
output/rl/bc/q4_expert.npz output/rl/bc/q4_bc.pt output/rl/q4_ppo_bc_init.zip
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from stable_baselines3 import PPO
|
||||
|
||||
from mcm26b.config import ProblemConfig
|
||||
from mcm26b.planner import safe_disk_radius, select_measurement_point
|
||||
from mcm26b.rl._constants import NUM_ACTIONS, OBS_DIM
|
||||
from mcm26b.rl.env import LocalizationEnv
|
||||
from mcm26b.rl.features import candidate_actions
|
||||
|
||||
HIDDEN = (128, 128)
|
||||
|
||||
|
||||
def expert_action(env: LocalizationEnv) -> int:
|
||||
"""当前状态的专家动作编号（与训练环境候选同源，就近映射）。"""
|
||||
config = env.config
|
||||
channel = env._channel
|
||||
region = env._belief.candidate_set(channel)
|
||||
position = env._client.state.position
|
||||
blocked = env._blocked
|
||||
candidates = candidate_actions(region, position, config, blocked)
|
||||
|
||||
if env.problem == 3:
|
||||
expert_point = select_measurement_point(
|
||||
region, position, config).position
|
||||
else:
|
||||
center, radius = region.mec()
|
||||
safe = safe_disk_radius(region, config)
|
||||
if not blocked:
|
||||
expert_point = select_measurement_point(
|
||||
region, position, config).position
|
||||
elif len(blocked) == 1 and safe >= 0:
|
||||
first = blocked[0]
|
||||
dx, dy = center[0] - first[0], center[1] - first[1]
|
||||
norm = math.hypot(dx, dy)
|
||||
if norm <= 1e-9:
|
||||
expert_point = (center[0] + safe, center[1])
|
||||
else:
|
||||
scale = min(1.0, safe / norm)
|
||||
expert_point = (center[0] + dx * scale,
|
||||
center[1] + dy * scale)
|
||||
else:
|
||||
first = blocked[0]
|
||||
base = math.atan2(first[1] - center[1], first[0] - center[0])
|
||||
angle = base + math.radians(45.0 * (len(blocked) - 1))
|
||||
expert_point = (center[0] + radius * math.cos(angle),
|
||||
center[1] + radius * math.sin(angle))
|
||||
return min(range(len(candidates)),
|
||||
key=lambda k: math.dist(candidates[k], expert_point))
|
||||
|
||||
|
||||
class BCMlp(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.net = torch.nn.Sequential(
|
||||
torch.nn.Linear(OBS_DIM, HIDDEN[0]), torch.nn.Tanh(),
|
||||
torch.nn.Linear(HIDDEN[0], HIDDEN[1]), torch.nn.Tanh(),
|
||||
torch.nn.Linear(HIDDEN[1], NUM_ACTIONS))
|
||||
|
||||
def forward(self, x):
|
||||
return self.net(x)
|
||||
|
||||
|
||||
def collect(problem: int, episodes: int, seed: int, config: ProblemConfig):
|
||||
env = LocalizationEnv(
|
||||
config, problem=problem,
|
||||
directional_probability=1.0 if problem == 4 else 0.0,
|
||||
max_episode_steps=16)
|
||||
observations, actions, groups = [], [], []
|
||||
for episode in range(episodes):
|
||||
obs, _info = env.reset(seed=seed + episode)
|
||||
for _step in range(env.max_episode_steps + 2):
|
||||
action = expert_action(env)
|
||||
observations.append(obs)
|
||||
actions.append(action)
|
||||
groups.append(episode)
|
||||
obs, _reward, terminated, truncated, _info = env.step(action)
|
||||
if terminated or truncated:
|
||||
break
|
||||
return np.asarray(observations, dtype=np.float32), \
|
||||
np.asarray(actions, dtype=np.int64), np.asarray(groups, dtype=np.int64)
|
||||
|
||||
|
||||
def train_bc(x, y, *, epochs: int, batch_size: int, lr: float,
|
||||
device: str, groups, valid_frac: float = 0.1):
|
||||
model = BCMlp().to(device)
|
||||
optimizer = torch.optim.AdamW(model.parameters(), lr=lr)
|
||||
episodes = np.unique(groups)
|
||||
if len(episodes) < 2:
|
||||
raise ValueError('行为克隆至少需要两个独立episode才能划分验证集')
|
||||
valid_groups = np.random.default_rng(0).permutation(episodes)
|
||||
valid_groups = valid_groups[:max(1, int(len(episodes)*valid_frac))]
|
||||
valid = torch.as_tensor(np.isin(groups, valid_groups), device=device)
|
||||
x_t, y_t = torch.as_tensor(x, device=device), torch.as_tensor(y, device=device)
|
||||
x_v, y_v = x_t[valid], y_t[valid]
|
||||
x_tr, y_tr = x_t[~valid], y_t[~valid]
|
||||
loss_fn = torch.nn.CrossEntropyLoss()
|
||||
for epoch in range(epochs):
|
||||
order = torch.randperm(len(x_tr), device=device)
|
||||
total = correct = 0.0
|
||||
for start in range(0, len(order), batch_size):
|
||||
idx = order[start:start + batch_size]
|
||||
logits = model(x_tr[idx])
|
||||
loss = loss_fn(logits, y_tr[idx])
|
||||
optimizer.zero_grad()
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
total += float(loss.item()) * len(idx)
|
||||
correct += int((logits.argmax(1) == y_tr[idx]).sum().item())
|
||||
train_acc = correct / len(x_tr)
|
||||
with torch.no_grad():
|
||||
valid_acc = float((model(x_v).argmax(1) == y_v).float().mean().item())
|
||||
if epoch == 0 or (epoch + 1) % 5 == 0:
|
||||
print(json.dumps({"event": "bc_epoch", "epoch": epoch + 1,
|
||||
"loss": round(total / len(x_tr), 4),
|
||||
"train_acc": round(train_acc, 4),
|
||||
"valid_acc": round(valid_acc, 4)},
|
||||
ensure_ascii=False))
|
||||
return model, float(valid_acc)
|
||||
|
||||
|
||||
def copy_into_ppo(problem: int, model: BCMlp, device: str) -> Path | None:
|
||||
"""把 BC 网络权重按 SB3 的 policy_net/action_net 拆分拷入 PPO。"""
|
||||
# 当前奖励/特征已修订，重新初始化critic，不能沿用旧任务的价值预测。
|
||||
env = LocalizationEnv(problem=problem)
|
||||
ppo = PPO('MlpPolicy', env, policy_kwargs={'net_arch': list(HIDDEN)},
|
||||
n_steps=512, batch_size=128, gamma=1.0, device=device)
|
||||
state = {key.removeprefix("net."): value
|
||||
for key, value in model.state_dict().items()}
|
||||
# SB3 MlpPolicy：policy_net 只含前两个 Linear（Tanh 激活），
|
||||
# 输出层在 action_net。BC 的 Sequential 键为 0/2/4。
|
||||
policy_state = {key: value for key, value in state.items()
|
||||
if key in ("0.weight", "0.bias", "2.weight", "2.bias")}
|
||||
action_state = {
|
||||
"weight": state["4.weight"], "bias": state["4.bias"]}
|
||||
ppo.policy.mlp_extractor.policy_net.load_state_dict(policy_state)
|
||||
ppo.policy.action_net.load_state_dict(action_state)
|
||||
# 数值检查整个前向映射，而不只是线性层形状。
|
||||
probe = torch.randn((8, OBS_DIM), device=device)
|
||||
with torch.no_grad():
|
||||
copied = ppo.policy.action_net(ppo.policy.mlp_extractor.policy_net(probe))
|
||||
if not torch.allclose(model(probe), copied, atol=1e-6):
|
||||
raise RuntimeError('行为克隆与PPO激活函数/前向映射不一致')
|
||||
init_path = Path(f"output/rl/q{problem}_ppo_bc_init.zip")
|
||||
ppo.save(str(init_path))
|
||||
env.close()
|
||||
init_path.with_suffix('.json').write_text(json.dumps(
|
||||
dict(algorithm='ppo', feature_version=2, problem=problem)), encoding='utf-8')
|
||||
return init_path
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--problem", type=int, choices=(3, 4), required=True)
|
||||
parser.add_argument("--episodes", type=int, default=3000)
|
||||
parser.add_argument("--epochs", type=int, default=20)
|
||||
parser.add_argument("--batch-size", type=int, default=256)
|
||||
parser.add_argument("--lr", type=float, default=3e-4)
|
||||
parser.add_argument("--seed", type=int, default=20260911)
|
||||
parser.add_argument("--device", default="auto")
|
||||
parser.add_argument("--copy-only", action="store_true",
|
||||
help="跳过采集/训练，只把已有 q<problem>_bc.pt 拷入 PPO 初始化")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if args.device == "auto":
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
else:
|
||||
device = args.device
|
||||
print(json.dumps({"event": "bc_start", "problem": args.problem,
|
||||
"episodes": args.episodes, "device": device,
|
||||
"cuda": torch.cuda.is_available()},
|
||||
ensure_ascii=False))
|
||||
|
||||
if args.copy_only:
|
||||
bc_dir = Path("output/rl/bc")
|
||||
checkpoint = torch.load(bc_dir / f"q{args.problem}_bc.pt",
|
||||
map_location="cpu")
|
||||
if checkpoint.get('activation') != 'Tanh' or checkpoint.get('feature_version') != 2:
|
||||
parser.error('旧ReLU/旧特征BC检查点不能直接拷贝，请重新采集并训练')
|
||||
model = BCMlp()
|
||||
model.load_state_dict(checkpoint["state_dict"])
|
||||
init_path = copy_into_ppo(args.problem, model, device)
|
||||
dataset = np.load(bc_dir / f"q{args.problem}_expert.npz",
|
||||
allow_pickle=False)
|
||||
report = {
|
||||
"problem": args.problem,
|
||||
"samples": int(len(dataset["obs"])),
|
||||
"valid_accuracy": checkpoint.get("valid_acc"),
|
||||
"dataset": str(bc_dir / f"q{args.problem}_expert.npz"),
|
||||
"bc_model": str(bc_dir / f"q{args.problem}_bc.pt"),
|
||||
"ppo_init": str(init_path) if init_path is not None else None,
|
||||
}
|
||||
(bc_dir / f"q{args.problem}_bc.json").write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print(json.dumps({"event": "bc_copy", "problem": args.problem,
|
||||
"ppo_init": str(init_path), **report},
|
||||
ensure_ascii=False, indent=2))
|
||||
return 0 if init_path is not None else 2
|
||||
|
||||
config = ProblemConfig()
|
||||
started = time.monotonic()
|
||||
torch.manual_seed(args.seed)
|
||||
x, y, groups = collect(args.problem, args.episodes, args.seed, config)
|
||||
collect_s = time.monotonic() - started
|
||||
bc_dir = Path("output/rl/bc")
|
||||
bc_dir.mkdir(parents=True, exist_ok=True)
|
||||
np.savez_compressed(bc_dir / f"q{args.problem}_expert.npz", obs=x, action=y, episode=groups)
|
||||
|
||||
model, valid_acc = train_bc(x, y, epochs=args.epochs,
|
||||
batch_size=args.batch_size, lr=args.lr,
|
||||
device=device, groups=groups)
|
||||
torch.save({"state_dict": model.state_dict(), "obs_dim": OBS_DIM,
|
||||
"actions": NUM_ACTIONS, "valid_acc": valid_acc,
|
||||
"activation": "Tanh", "feature_version": 2},
|
||||
bc_dir / f"q{args.problem}_bc.pt")
|
||||
|
||||
# 拷入现有 PPO 的 policy_net/action_net 作为初始化（critic 保持原权重）。
|
||||
init_path = copy_into_ppo(args.problem, model, device)
|
||||
report = {
|
||||
"problem": args.problem, "episodes": args.episodes,
|
||||
"samples": int(len(x)), "valid_accuracy": valid_acc,
|
||||
"collect_seconds": round(collect_s, 1),
|
||||
"train_seconds": round(time.monotonic() - started - collect_s, 1),
|
||||
"dataset": str(bc_dir / f"q{args.problem}_expert.npz"),
|
||||
"bc_model": str(bc_dir / f"q{args.problem}_bc.pt"),
|
||||
"ppo_init": str(init_path) if init_path is not None else None,
|
||||
}
|
||||
(bc_dir / f"q{args.problem}_bc.json").write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,36 +0,0 @@
|
||||
"""从 Tee 输出的 CLI 文本中提取末尾 JSON，落盘为 summary.json。
|
||||
|
||||
官方演练目录由 run 命令产生 cli-output.txt（前面可能有 CUDA 警告），
|
||||
本脚本只提取最后一个完整 JSON 对象，并核对 adjacent 的官方 result.json。
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def extract(text: str) -> dict:
|
||||
start = text.rfind("\n{")
|
||||
if start < 0:
|
||||
raise ValueError("未找到 JSON 对象")
|
||||
return json.loads(text[start + 1:])
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("folder", type=Path)
|
||||
args = parser.parse_args()
|
||||
folder = args.folder
|
||||
summary = extract((folder / "cli-output.txt").read_text(encoding="utf-8"))
|
||||
(folder / "summary.json").write_text(
|
||||
json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({
|
||||
"folder": str(folder), "completed": summary["completed"],
|
||||
"virtual_time_s": summary["virtual_time_s"],
|
||||
"cleared_channels": summary["cleared_channels"],
|
||||
"total_requests": summary["total_requests"],
|
||||
}, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,78 +0,0 @@
|
||||
"""汇总配对置信区间、历史官方证据与BC/特征错位探针，不连接模拟器。"""
|
||||
from pathlib import Path
|
||||
import json
|
||||
import hashlib
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
import verify_rl_practice
|
||||
from mcm26b.rl.env import LocalizationEnv
|
||||
from mcm26b.rl.features import build_observation
|
||||
|
||||
|
||||
def main():
|
||||
root = Path('output/rl/review-20260911')
|
||||
records = json.loads((root/'evaluation/validation/final/results.json')
|
||||
.read_text(encoding='utf-8'))
|
||||
rng = np.random.default_rng(20260911)
|
||||
comparisons = {}
|
||||
for problem in (3,4):
|
||||
part = [r for r in records['rows'] if r['problem']==problem]
|
||||
for label in ('baseline','legacy'):
|
||||
baseline = {r['case']:r for r in part if r['variant']==label}
|
||||
selected = sorted((r for r in part if r['variant']=='revised'),key=lambda r:r['case'])
|
||||
a = np.array([baseline[r['case']]['virtual_s'] for r in selected])
|
||||
b = np.array([r['virtual_s'] for r in selected])
|
||||
samples = rng.integers(0,len(a),(10000,len(a)))
|
||||
changes = (b[samples].mean(axis=1)/a[samples].mean(axis=1)-1)*100
|
||||
comparisons[f'q{problem}-revised-vs-{label}'] = dict(
|
||||
relative_change_pct=(b.mean()/a.mean()-1)*100,
|
||||
paired_bootstrap_95_pct=np.quantile(changes,[.025,.975]).tolist(),
|
||||
faster_cases=int(sum(b<a)),runs=len(a))
|
||||
# 重新计算原始官方日志的检查，派生报告单独保存，原始归档不写入。
|
||||
verify_rl_practice.save_json = lambda path,value: None
|
||||
official = [verify_rl_practice.audit(p.parent) for p in sorted(
|
||||
Path('output/rl/official-practice').rglob('*.result.json'))]
|
||||
env = LocalizationEnv(handoff_probability=0,boundary_probability=0)
|
||||
fixed,_ = env.reset(seed=1)
|
||||
# 复现旧版把整个BeliefState作为单频道对象传入的错误。
|
||||
wrong = build_observation(env._belief.candidate_set(env._channel),env._client.state.position,
|
||||
env._channel,env.config,env.problem,env._blocked,env._belief,
|
||||
virtual_time_s=env._client.state.virtual_time_s)
|
||||
features = dict(mismatched_indices=np.flatnonzero(fixed!=wrong).tolist(),
|
||||
fixed_history_count=float(fixed[7]),old_history_count=float(wrong[7]))
|
||||
env.close()
|
||||
torch.set_num_threads(1)
|
||||
torch.manual_seed(0)
|
||||
bc = {}
|
||||
for problem in (3,4):
|
||||
# 使用旧检查点和原始专家输入，比对原ReLU网络与拷贝后的Tanh网络。
|
||||
checkpoint = torch.load(f'output/rl/bc/q{problem}_bc.pt',map_location='cpu',
|
||||
weights_only=True)
|
||||
data = np.load(f'output/rl/bc/q{problem}_expert.npz',allow_pickle=False)
|
||||
x = torch.from_numpy(data['obs'][::max(1,len(data['obs'])//1000)].copy())
|
||||
|
||||
def network(activation, checkpoint=checkpoint):
|
||||
net = torch.nn.Sequential(torch.nn.Linear(401,128),activation(),
|
||||
torch.nn.Linear(128,128),activation(),torch.nn.Linear(128,60))
|
||||
state = {k.removeprefix('net.'):v for k,v in checkpoint['state_dict'].items()}
|
||||
net.load_state_dict(state)
|
||||
return net
|
||||
with torch.no_grad():
|
||||
before,after=network(torch.nn.ReLU)(x),network(torch.nn.Tanh)(x)
|
||||
bc[f'q{problem}'] = dict(samples=len(x),
|
||||
different_argmax_fraction=float((before.argmax(1)!=after.argmax(1)).float().mean()),
|
||||
max_logit_difference=float((before-after).abs().max()))
|
||||
report = dict(comparisons=comparisons,official=official,feature_probe=features,bc_probe=bc,
|
||||
limitations='置信区间仅重采样固定模型的场景，不覆盖训练随机种子不确定性。',
|
||||
code_sha256={str(p):hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
for p in sorted(Path('src/mcm26b/rl').glob('*.py'))})
|
||||
(root/'review_evidence.json').write_text(
|
||||
json.dumps(report,ensure_ascii=False,indent=2),encoding='utf-8')
|
||||
print(json.dumps(dict(comparisons=comparisons,feature_probe=features,bc_probe=bc,
|
||||
official=[dict(case=r['case_code'],passed=r['passed'])
|
||||
for r in official]),indent=2))
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
@@ -1,987 +0,0 @@
|
||||
"""高层 MaskablePPO 训练（P1）。
|
||||
|
||||
架构与边界：
|
||||
- 环境 = :class:`mcm26b.hrl.hierarchical_env.HierarchicalEnv`，每一步是一个高层
|
||||
宏动作（选择 0–119 槽位：clear/locate/station/fallback/保留）；
|
||||
- 训练/部署共用 :mod:`mcm26b.hrl.scheduler` 的候选、观测和动作掩码；
|
||||
- 低层默认使用几何选点；``--local-selector rl`` 仅作为接口预留并记录，
|
||||
当前 HTTP 执行路径仍由高层宏动作终点决定，不把八方向推演当正式收益。
|
||||
|
||||
落盘保证（长训必需）：
|
||||
- **急停可恢复**：``--recovery-every``（默认 2000 环境步）把一份滚动恢复点
|
||||
``<模型目录>/checkpoints/<stem>_latest.zip`` 原子覆盖写；训练开始前先写一份
|
||||
起步基线。因此即使进程被 SIGKILL / ``taskkill /F`` 强杀（没有任何 Python
|
||||
收尾机会），也**最多只丢 ``--recovery-every`` 个环境步**，且磁盘上始终有一份
|
||||
完整、可加载、可 ``--resume`` 的产物；
|
||||
- **原子落盘**：模型 zip、旁挂 json 与 ``<run_dir>/resume.json`` 都是"临时文件 +
|
||||
``os.replace``"，保存途中被杀不会留下半截文件，也不会破坏上一份完整快照；
|
||||
- ``--checkpoint-every`` 每 N 个环境步在 ``<模型目录>/checkpoints/`` 另留一份历史
|
||||
checkpoint（``<stem>_<步数>.zip``），只保留最近 ``--checkpoint-keep`` 个，供
|
||||
返祖对比与多档恢复；
|
||||
- ``<run_dir>/resume.json`` 是恢复的**权威指针**：无论上一刻是滚动恢复点、历史
|
||||
checkpoint、优雅收尾还是异常兜底，它总指向一份可续训产物，并记录增量语义
|
||||
（``timesteps`` 是追加步数）与生效过的 PPO 超参；
|
||||
- ``--control-file`` 是训练前端的**跨平台**控制通道：前端原子写 JSON
|
||||
（``paused`` / ``stop``），本脚本每个 step 轮询。暂停时训练在 step 边界挂起，
|
||||
停止时先保存模型与元数据再退出；
|
||||
- 信号（SIGTERM/SIGINT，Windows 上另有 SIGBREAK）仍可用：直接 kill 本脚本或
|
||||
前端发进程组信号时同样走优雅收尾；subproc worker 在 :func:`_make_env` 里忽略
|
||||
这些信号，保证它们活到主进程 ``vec_env.close()`` 之后再退出；
|
||||
- 任何异常都会先落一份 ``*_emergency.zip`` + ``resume.json`` 再抛出，不再出现
|
||||
"跑了很久什么都没有"。
|
||||
|
||||
为什么暂停/停止不靠信号：Windows 没有 SIGSTOP，且 ``os.kill(pid, SIGTERM)`` 会
|
||||
退化成 ``TerminateProcess`` 硬杀——收尾分支根本不会执行。控制文件两端都只是
|
||||
普通文件 IO，POSIX 与 Windows 行为一致（见 :mod:`mcm26b.process_control`）。
|
||||
|
||||
P1 smoke 示例：
|
||||
python scripts/train_hrl_high.py \
|
||||
--problem 3 --stage 1 --timesteps 2048 --seed 320001 \
|
||||
--envs 1 --vec-env dummy --device mps \
|
||||
--config-profile production --model output/hrl/p1-smoke/q3.zip \
|
||||
--eval-episodes 2 --verbose 0
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import platform
|
||||
import random
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from mcm26b.config import ProblemConfig
|
||||
from mcm26b.hrl.curriculum import STAGES
|
||||
from mcm26b.hrl.constants import (ACTION_LAYOUT_VERSION, FEATURE_VERSION,
|
||||
HIGH_OBS_DIM, N_CANDIDATES, REWARD_VERSION)
|
||||
from mcm26b.process_control import ControlState, read_control
|
||||
|
||||
|
||||
def _stage_from_index(index: int):
|
||||
if not 1 <= int(index) <= len(STAGES):
|
||||
raise ValueError(f"stage 必须在 1..{len(STAGES)}")
|
||||
return STAGES[int(index) - 1]
|
||||
|
||||
|
||||
def _profile_fidelity(profile: str) -> str:
|
||||
if profile == "production":
|
||||
return "production"
|
||||
if profile == "coarse-pretrain":
|
||||
return "fast"
|
||||
raise ValueError(f"未知 config-profile: {profile}")
|
||||
|
||||
|
||||
def _reward_from_profile(profile: str):
|
||||
from mcm26b.hrl.hierarchical_env import HRLReward
|
||||
|
||||
if profile == "time":
|
||||
return HRLReward.time_only()
|
||||
if profile == "prompt":
|
||||
return HRLReward()
|
||||
raise ValueError(f"未知 reward-profile: {profile}")
|
||||
|
||||
|
||||
def _stop_signals() -> list[int]:
|
||||
"""本平台可用于"请求收尾"的信号集合。
|
||||
|
||||
Windows 上没有 SIGTERM 的软语义（``os.kill`` 直接 TerminateProcess），但
|
||||
``SIGBREAK`` 可以由 ``CTRL_BREAK_EVENT`` 触发并进入 Python 处理器；POSIX 上
|
||||
SIGTERM/SIGINT 均可捕获。控制文件才是主通道，这里只是让"直接 kill 脚本"
|
||||
也尽量走优雅收尾。
|
||||
"""
|
||||
signals = [signal.SIGTERM, signal.SIGINT]
|
||||
sigbreak = getattr(signal, "SIGBREAK", None)
|
||||
if sigbreak is not None:
|
||||
signals.append(sigbreak)
|
||||
return signals
|
||||
|
||||
|
||||
def _ignore_stop_signals_in_worker() -> None:
|
||||
"""subproc worker 忽略收尾信号，保证主进程能优雅落盘。
|
||||
|
||||
无论信号来自进程组 SIGTERM 还是 Windows ``CTRL_BREAK_EVENT``，若 worker
|
||||
同时退出，主进程会在保存模型之前就拿到断管/EOF 错误，整局训练白跑。worker
|
||||
活在管道后面，主进程收尾时通过 ``vec_env.close()`` 的 ``close`` 指令正常
|
||||
结束它们。
|
||||
"""
|
||||
try:
|
||||
import multiprocessing
|
||||
if multiprocessing.current_process().name == "MainProcess":
|
||||
return
|
||||
for signum in _stop_signals():
|
||||
signal.signal(signum, signal.SIG_IGN)
|
||||
except (ImportError, ValueError, OSError): # 非主线程/平台不支持：不影响训练
|
||||
return
|
||||
|
||||
|
||||
class _ControlPoll:
|
||||
"""控制文件轮询状态机（纯标准库，可脱离 SB3 单测）。
|
||||
|
||||
``poll()`` 每 ``poll_every`` 次调用才真正读盘，避免每个 env step 都做一次
|
||||
文件 IO；``wait_while_paused()`` 阻塞到暂停解除或收到 stop，并累计暂停秒数
|
||||
（``--max-wall-s`` 要把这段扣掉，否则"挂起一小时回来就判超时"）。
|
||||
"""
|
||||
|
||||
def __init__(self, path: Path, *, poll_every: int = 32,
|
||||
poll_s: float = 0.2):
|
||||
self.path = Path(path)
|
||||
self.poll_every = max(1, int(poll_every))
|
||||
self.poll_s = max(0.02, float(poll_s))
|
||||
self.paused_total_s = 0.0
|
||||
self._calls = 0
|
||||
self._state = ControlState()
|
||||
|
||||
def poll(self) -> ControlState:
|
||||
"""按抽样间隔返回最近一次控制状态。"""
|
||||
self._calls += 1
|
||||
if self._calls % self.poll_every:
|
||||
return self._state
|
||||
self._state = read_control(self.path)
|
||||
return self._state
|
||||
|
||||
def wait_while_paused(self, on_pause, on_resume) -> bool:
|
||||
"""阻塞到继续或停止；返回 True 表示期间收到了 stop。
|
||||
|
||||
``on_pause`` / ``on_resume`` 用来向前端回显事件（训练脚本的 JSON 行）。
|
||||
"""
|
||||
on_pause()
|
||||
started = time.monotonic()
|
||||
stopped = False
|
||||
while True:
|
||||
time.sleep(self.poll_s)
|
||||
self._state = read_control(self.path)
|
||||
if self._state.stop:
|
||||
stopped = True
|
||||
break
|
||||
if not self._state.paused:
|
||||
break
|
||||
self.paused_total_s += time.monotonic() - started
|
||||
on_resume()
|
||||
return stopped
|
||||
|
||||
|
||||
def _make_env(problem: int, stage_index: int,
|
||||
monitor_path: str | None, profile: str,
|
||||
reward_profile: str):
|
||||
"""顶层工厂：供 DummyVecEnv / SubprocVecEnv 调用（必须是可 pickle 的）。"""
|
||||
from stable_baselines3.common.monitor import Monitor
|
||||
|
||||
from mcm26b.hrl.hierarchical_env import HierarchicalEnv
|
||||
|
||||
_ignore_stop_signals_in_worker()
|
||||
stage = _stage_from_index(stage_index)
|
||||
env = HierarchicalEnv(
|
||||
problem=problem,
|
||||
stage=stage,
|
||||
fidelity=_profile_fidelity(profile),
|
||||
max_episode_steps=300,
|
||||
real_budget_s=1800.0,
|
||||
reward=_reward_from_profile(reward_profile),
|
||||
)
|
||||
return Monitor(env, filename=monitor_path)
|
||||
|
||||
|
||||
def _build_vec_env(args, monitor_paths: list[str]):
|
||||
from stable_baselines3.common.vec_env import (DummyVecEnv, SubprocVecEnv)
|
||||
|
||||
from functools import partial
|
||||
|
||||
factories = [
|
||||
partial(_make_env, args.problem, args.stage,
|
||||
monitor_paths[rank], args.config_profile, args.reward_profile)
|
||||
for rank in range(args.envs)
|
||||
]
|
||||
if args.vec_env == "subproc":
|
||||
# 每个 worker 独立 Monitor 文件；spawn 入口在 __main__ 保护内。
|
||||
return SubprocVecEnv(factories, start_method="spawn")
|
||||
return DummyVecEnv(factories)
|
||||
|
||||
|
||||
def _hash_file(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def _policy_hyperparams(model) -> dict[str, Any]:
|
||||
"""记录**实际生效**的 PPO 超参。
|
||||
|
||||
``--resume``/``--init`` 时 ``MaskablePPO.load`` 会一并恢复 checkpoint 内的
|
||||
n_steps/batch_size/n_epochs 等，命令行同名参数被忽略；把它们写进元数据，
|
||||
事后才能核对一局到底用什么配置训的。
|
||||
"""
|
||||
|
||||
def _scalar(value):
|
||||
# SB3 2.9 的 learning_rate/clip_range 可能是 Schedule 对象
|
||||
current = getattr(value, "current_value", None)
|
||||
if callable(current):
|
||||
try:
|
||||
value = current()
|
||||
except Exception:
|
||||
return str(value)
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return str(value)
|
||||
|
||||
return {
|
||||
"n_steps": int(getattr(model, "n_steps", 0) or 0),
|
||||
"batch_size": int(getattr(model, "batch_size", 0) or 0),
|
||||
"n_epochs": int(getattr(model, "n_epochs", 0) or 0),
|
||||
"gamma": _scalar(getattr(model, "gamma", 0.0)),
|
||||
"gae_lambda": _scalar(getattr(model, "gae_lambda", 0.0)),
|
||||
"learning_rate": _scalar(getattr(model, "learning_rate", 0.0)),
|
||||
"clip_range": _scalar(getattr(model, "clip_range", 0.0)),
|
||||
"ent_coef": _scalar(getattr(model, "ent_coef", 0.0)),
|
||||
"vf_coef": _scalar(getattr(model, "vf_coef", 0.0)),
|
||||
"max_grad_norm": _scalar(getattr(model, "max_grad_norm", 0.0)),
|
||||
}
|
||||
|
||||
|
||||
def _progress_payload(model, start: int) -> dict[str, Any]:
|
||||
"""组装 ``--progress-json`` 的单行负载。
|
||||
|
||||
纯函数（只读 model 的 logger / ep_info_buffer），便于脱离训练循环测试；
|
||||
NaN / ±inf 一律折算成 None，保证 ``json.dumps(allow_nan=False)`` 不会炸。
|
||||
"""
|
||||
values = getattr(model.logger, "name_to_value", {}) or {}
|
||||
|
||||
def _num(key, default=None):
|
||||
value = values.get(key)
|
||||
if value is None:
|
||||
return default
|
||||
try:
|
||||
value = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
if not math.isfinite(value):
|
||||
return default
|
||||
return value
|
||||
|
||||
ep_rewards: list[float] = []
|
||||
ep_lengths: list[float] = []
|
||||
for item in getattr(model, "ep_info_buffer", []) or []:
|
||||
try:
|
||||
ep_rewards.append(float(item.get("r")))
|
||||
ep_lengths.append(float(item.get("l")))
|
||||
except (TypeError, ValueError, AttributeError):
|
||||
continue
|
||||
return {
|
||||
"event": "progress",
|
||||
"timesteps": int(model.num_timesteps),
|
||||
"start_timesteps": int(start),
|
||||
"n_updates": int(getattr(model, "_n_updates", 0) or 0),
|
||||
"ep_rew_mean": (sum(ep_rewards) / len(ep_rewards)
|
||||
if ep_rewards else None),
|
||||
"ep_len_mean": (sum(ep_lengths) / len(ep_lengths)
|
||||
if ep_lengths else None),
|
||||
"value_loss": _num("train/value_loss"),
|
||||
"policy_gradient_loss": _num("train/policy_gradient_loss"),
|
||||
"approx_kl": _num("train/approx_kl"),
|
||||
"entropy_loss": _num("train/entropy_loss"),
|
||||
"explained_variance": _num("train/explained_variance"),
|
||||
"fps": _num("time/fps"),
|
||||
}
|
||||
|
||||
|
||||
def _model_metadata(*, args, wall_s: float, n_envs: int,
|
||||
actual_timesteps: int) -> dict[str, Any]:
|
||||
import numpy
|
||||
import stable_baselines3
|
||||
import sb3_contrib
|
||||
import torch
|
||||
|
||||
stage = _stage_from_index(args.stage)
|
||||
return {
|
||||
"level": "high",
|
||||
"problem": int(args.problem),
|
||||
"stage": int(args.stage),
|
||||
"stage_name": stage.name,
|
||||
"algorithm": "maskable_ppo",
|
||||
"feature_version": FEATURE_VERSION,
|
||||
"action_layout_version": ACTION_LAYOUT_VERSION,
|
||||
"reward_version": REWARD_VERSION,
|
||||
"obs_dim": HIGH_OBS_DIM,
|
||||
"act_dim": N_CANDIDATES,
|
||||
"seed": int(args.seed),
|
||||
"requested_timesteps": int(args.timesteps),
|
||||
"timesteps_semantics": ("additional" if (args.resume or args.init)
|
||||
else "total"),
|
||||
"actual_timesteps": int(actual_timesteps),
|
||||
"n_envs": int(n_envs),
|
||||
"vec_env": args.vec_env,
|
||||
"config_profile": args.config_profile,
|
||||
"fidelity": _profile_fidelity(args.config_profile),
|
||||
"reward_profile": args.reward_profile,
|
||||
"local_selector": args.local_selector,
|
||||
"local_model": args.local_model,
|
||||
"checkpoint_every": int(getattr(args, "checkpoint_every", 0) or 0),
|
||||
"checkpoint_keep": int(getattr(args, "checkpoint_keep", 0) or 0),
|
||||
"checkpoint_dir": getattr(args, "checkpoint_dir", None),
|
||||
"recovery_every": int(getattr(args, "recovery_every", 0) or 0),
|
||||
"device": args.device,
|
||||
"wall_seconds": round(wall_s, 3),
|
||||
"max_wall_s": float(args.max_wall_s),
|
||||
"created_unix_s": time.time(),
|
||||
"python": sys.version.split()[0],
|
||||
"platform": platform.platform(),
|
||||
"torch": torch.__version__,
|
||||
"numpy": numpy.__version__,
|
||||
"stable_baselines3": stable_baselines3.__version__,
|
||||
"sb3_contrib": sb3_contrib.__version__,
|
||||
"mps_available": bool(getattr(getattr(torch.backends, "mps", None),
|
||||
"is_available", lambda: False)()),
|
||||
}
|
||||
|
||||
|
||||
def _write_json_atomic(path: Path, payload: dict[str, Any]) -> None:
|
||||
"""原子写 JSON：同目录 ``.tmp-`` 前缀临时文件 + ``os.replace``。
|
||||
|
||||
急停（SIGKILL / ``taskkill /F``）会绕过所有 Python 清理逻辑，保存途中被杀
|
||||
是完全可能的。原子替换保证**真实文件名下要么是旧的完整文件、要么是新的
|
||||
完整文件**，不会出现半截 JSON 让续训读到一半就崩。
|
||||
"""
|
||||
path = Path(path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_name(f".tmp-{path.name}")
|
||||
tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8")
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
def _save_model_atomic(model, target: Path) -> Path:
|
||||
"""把 SB3 模型原子地写到 ``target``，返回实际落盘路径。
|
||||
|
||||
先写 ``.tmp-<name>``（同样以 ``.zip`` 结尾，SB3 不会再加后缀），完整写完后
|
||||
``os.replace`` 覆盖目标。于是：
|
||||
|
||||
- 保存途中被强杀 → 目标路径仍是上一份**完整**的快照，可继续复训；
|
||||
- 保存成功 → 目标路径立即变成新快照，不存在"看起来存在但打不开"的窗口。
|
||||
|
||||
临时名带 ``.tmp-`` 前缀，因此不会被 ``*_ckpt_*.zip`` 的清理逻辑误当成正式
|
||||
checkpoint。
|
||||
"""
|
||||
target = Path(target)
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = target.with_name(f".tmp-{target.name}")
|
||||
model.save(str(tmp))
|
||||
if not tmp.exists(): # SB3 对未知后缀可能自行补 .zip
|
||||
candidate = tmp.with_name(tmp.name + ".zip")
|
||||
if candidate.exists():
|
||||
tmp = candidate
|
||||
os.replace(tmp, target)
|
||||
return target
|
||||
|
||||
|
||||
def _write_resume_hint(*, run_dir: Path, model_path: Path, timesteps: int,
|
||||
reason: str, model=None) -> dict[str, Any]:
|
||||
"""把"从哪继续"写进 ``<run_dir>/resume.json``。
|
||||
|
||||
这是急停恢复的**权威指针**：无论上一刻是周期 checkpoint、滚动恢复点、
|
||||
优雅收尾还是异常兜底，这个文件总指向一份完整、可 ``--resume`` 的产物，
|
||||
并带上续训语义（``timesteps`` 是增量步数）与生效过的 PPO 超参。
|
||||
"""
|
||||
hint: dict[str, Any] = {
|
||||
"event": "resume_hint",
|
||||
"model": str(model_path),
|
||||
"timesteps": int(timesteps),
|
||||
"timesteps_semantics": "additional",
|
||||
"reason": str(reason),
|
||||
"created_unix_s": time.time(),
|
||||
"resume_args": ["--resume", "--model", str(model_path)],
|
||||
}
|
||||
if model is not None:
|
||||
hint["ppo_hyperparams"] = _policy_hyperparams(model)
|
||||
_write_json_atomic(run_dir / "resume.json", hint)
|
||||
return hint
|
||||
|
||||
|
||||
def _write_metadata(*, model_path: Path, args, wall_s: float, n_envs: int,
|
||||
actual_timesteps: int, monitor_paths: list[str],
|
||||
model=None,
|
||||
extra: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
"""写模型旁挂元数据（含 sha256），训练前端/审计都读它。"""
|
||||
metadata = _model_metadata(args=args, wall_s=wall_s, n_envs=n_envs,
|
||||
actual_timesteps=actual_timesteps)
|
||||
if model is not None:
|
||||
metadata["ppo_hyperparams"] = _policy_hyperparams(model)
|
||||
metadata["model_sha256"] = _hash_file(model_path)
|
||||
metadata["monitor_files"] = list(monitor_paths)
|
||||
metadata["reward_note"] = (
|
||||
"reward_profile=time 使用 HRLReward.time_only（P0.4 纯时间口径）；"
|
||||
"reward_profile=prompt 使用原始 §5.2 塑形。模型元数据已固化口径。")
|
||||
if extra:
|
||||
metadata.update(extra)
|
||||
_write_json_atomic(model_path.with_suffix(".json"), metadata)
|
||||
return metadata
|
||||
|
||||
|
||||
def _prune_checkpoints(directory: Path, stem: str, keep: int) -> list[str]:
|
||||
"""只保留最近 ``keep`` 个 checkpoint（zip+json 同删），返回被删的 zip 路径。
|
||||
|
||||
只匹配 ``<stem>_<步数>.zip``；滚动恢复点 ``<stem>_latest.zip`` 与临时文件
|
||||
``.tmp-*.zip`` 都不匹配，因此永远不会被历史清理误删。
|
||||
"""
|
||||
files = sorted((path for path in directory.glob(f"{stem}_[0-9]*.zip")
|
||||
if path.stem.rsplit("_", 1)[-1].isdigit()),
|
||||
key=lambda path: path.stat().st_mtime, reverse=True)
|
||||
removed: list[str] = []
|
||||
for stale in files[max(1, int(keep)):]:
|
||||
for candidate in (stale, stale.with_suffix(".json")):
|
||||
if candidate.exists():
|
||||
candidate.unlink()
|
||||
removed.append(str(stale))
|
||||
return removed
|
||||
|
||||
|
||||
def _evaluate_env(model, *, problem: int, stage_index: int,
|
||||
episodes: int, seed: int, profile: str,
|
||||
reward_profile: str) -> dict[str, Any]:
|
||||
from mcm26b.hrl.hierarchical_env import HierarchicalEnv
|
||||
|
||||
stage = _stage_from_index(stage_index)
|
||||
env = HierarchicalEnv(problem=problem, stage=stage,
|
||||
fidelity=_profile_fidelity(profile),
|
||||
max_episode_steps=300, real_budget_s=1800.0,
|
||||
reward=_reward_from_profile(reward_profile))
|
||||
wins = 0
|
||||
rows = []
|
||||
if episodes <= 0:
|
||||
env.close()
|
||||
return {"episodes": 0, "successes": 0, "success_rate": None, "rows": []}
|
||||
for episode in range(episodes):
|
||||
obs, info = env.reset(seed=seed + episode)
|
||||
done = False
|
||||
steps = 0
|
||||
while not done:
|
||||
action, _state = model.predict(
|
||||
obs, action_masks=env.action_masks(), deterministic=True)
|
||||
obs, _, terminated, truncated, info = env.step(int(action))
|
||||
done = bool(terminated or truncated)
|
||||
steps += 1
|
||||
# 课程阶段的小场景只需证明全部频道 accounted；完整阶段至少应清掉
|
||||
# 场景最少的题面源数。这里不读取真值，只使用环境自身统计。
|
||||
success = bool(
|
||||
info.get("accounted", 0) >= 20
|
||||
and info.get("cleared", 0) >= stage.sources_min
|
||||
and not info.get("anomalies")
|
||||
)
|
||||
wins += int(success)
|
||||
rows.append({
|
||||
"episode": episode,
|
||||
"seed": seed + episode,
|
||||
"steps": steps,
|
||||
"cleared": info.get("cleared", 0),
|
||||
"accounted": info.get("accounted", 0),
|
||||
"virtual_time_s": info.get("episode_virtual_cost_s"),
|
||||
"stop_reason": info.get("stop_reason"),
|
||||
"anomalies": info.get("anomalies"),
|
||||
"success": success,
|
||||
})
|
||||
env.close()
|
||||
return {
|
||||
"episodes": len(rows),
|
||||
"successes": wins,
|
||||
"success_rate": wins / len(rows) if rows else 0.0,
|
||||
"rows": rows,
|
||||
}
|
||||
|
||||
|
||||
def _evaluate_manifest(model_path: Path, manifest_path: str, *,
|
||||
limit: int) -> dict[str, Any]:
|
||||
"""可选：用 benchmark_hrl 的同一套真值隔离评测在固定 manifest 上跑已训模型。"""
|
||||
scripts_dir = Path(__file__).resolve().parent
|
||||
if str(scripts_dir) not in sys.path:
|
||||
sys.path.insert(0, str(scripts_dir))
|
||||
import benchmark_hrl as bench # scripts/benchmark_hrl.py
|
||||
|
||||
manifest = json.loads(Path(manifest_path).read_text(encoding="utf-8"))
|
||||
scenes = bench.scenes_from_manifest(manifest)
|
||||
if limit > 0:
|
||||
scenes = scenes[:limit]
|
||||
rows = []
|
||||
for scene in scenes:
|
||||
run_dir = Path(model_path).parent / "eval-manifest" / scene.scene_id
|
||||
result = bench.run_scene_variant(
|
||||
scene, "hrl", config=ProblemConfig(),
|
||||
hrl_model=str(model_path), run_dir=run_dir)
|
||||
rows.append(result["row"])
|
||||
aggregate = bench._aggregate(rows)
|
||||
return aggregate
|
||||
|
||||
|
||||
def run_training(args) -> int:
|
||||
import numpy as np
|
||||
import torch
|
||||
from sb3_contrib import MaskablePPO
|
||||
from stable_baselines3.common.callbacks import BaseCallback
|
||||
from stable_baselines3.common.utils import set_random_seed
|
||||
|
||||
torch.set_num_threads(1)
|
||||
set_random_seed(args.seed)
|
||||
random.seed(args.seed)
|
||||
np.random.seed(args.seed)
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.manual_seed_all(args.seed)
|
||||
|
||||
model_path = Path(args.model)
|
||||
if model_path.exists() and not (args.resume or args.force):
|
||||
print(f"模型已存在：{model_path}；如需覆盖请加 --force，续训请加 --resume。",
|
||||
file=sys.stderr)
|
||||
return 2
|
||||
model_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
run_dir = model_path.parent
|
||||
monitor_paths = [
|
||||
str(run_dir / f"high_curve_{model_path.stem}_env{rank}.monitor.csv")
|
||||
for rank in range(args.envs)
|
||||
]
|
||||
for path in monitor_paths:
|
||||
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
vec_env = _build_vec_env(args, monitor_paths)
|
||||
|
||||
if args.resume:
|
||||
if not model_path.exists():
|
||||
print(f"--resume 指定的模型不存在：{model_path}", file=sys.stderr)
|
||||
return 2
|
||||
model = MaskablePPO.load(str(model_path), env=vec_env,
|
||||
device=args.device,
|
||||
print_system_info=False)
|
||||
elif args.init:
|
||||
init_path = Path(args.init)
|
||||
if not init_path.exists():
|
||||
print(f"--init 指定的模型不存在：{init_path}", file=sys.stderr)
|
||||
return 2
|
||||
model = MaskablePPO.load(str(init_path), env=vec_env,
|
||||
device=args.device,
|
||||
print_system_info=False)
|
||||
else:
|
||||
model = MaskablePPO(
|
||||
"MlpPolicy",
|
||||
vec_env,
|
||||
policy_kwargs={"net_arch": [128, 128]},
|
||||
n_steps=args.rollout_steps,
|
||||
batch_size=args.batch_size,
|
||||
n_epochs=args.n_epochs,
|
||||
gamma=1.0,
|
||||
gae_lambda=0.95,
|
||||
learning_rate=3e-4,
|
||||
clip_range=0.2,
|
||||
ent_coef=0.005,
|
||||
vf_coef=0.5,
|
||||
max_grad_norm=0.5,
|
||||
seed=args.seed,
|
||||
device=args.device,
|
||||
verbose=args.verbose,
|
||||
)
|
||||
if args.init:
|
||||
pass # 已在上面处理；此分支仅保持语义清晰
|
||||
|
||||
# resume/init 会恢复 checkpoint 内的 PPO 超参，命令行同名三项目前被忽略；
|
||||
# 显式说出差异，避免"以为换了 n_steps 其实没换"。
|
||||
if args.resume or args.init:
|
||||
effective = _policy_hyperparams(model)
|
||||
requested = {"n_steps": int(args.rollout_steps),
|
||||
"batch_size": int(args.batch_size),
|
||||
"n_epochs": int(args.n_epochs)}
|
||||
if {key: effective[key] for key in requested} != requested:
|
||||
print(json.dumps({
|
||||
"event": "hyperparams_from_checkpoint",
|
||||
"effective": {key: effective[key] for key in requested},
|
||||
"requested": requested,
|
||||
"note": ("resume/init 沿用 checkpoint 内的 PPO 超参，"
|
||||
"命令行 rollout-steps/batch-size/n-epochs 被忽略")},
|
||||
ensure_ascii=False), file=sys.stderr, flush=True)
|
||||
|
||||
# ------------------------------------------------------ 优雅暂停 / 停止
|
||||
# 主通道是控制文件（跨平台）；信号是兜底：直接 kill 本进程时也走同一段收尾。
|
||||
stop_state: dict[str, Any] = {"requested": False, "signal": None}
|
||||
previous_handlers: dict[int, Any] = {}
|
||||
|
||||
def _request_stop(source: str) -> None:
|
||||
if stop_state["requested"]:
|
||||
return
|
||||
stop_state["requested"] = True
|
||||
stop_state["signal"] = source
|
||||
print(json.dumps({"event": "stop_requested", "signal": source}), flush=True)
|
||||
|
||||
def _on_stop_signal(signum, _frame):
|
||||
_request_stop(signal.Signals(signum).name)
|
||||
|
||||
for _signum in _stop_signals():
|
||||
try:
|
||||
previous_handlers[_signum] = signal.signal(_signum, _on_stop_signal)
|
||||
except (ValueError, OSError): # 非主线程/平台不支持：不阻塞训练
|
||||
pass
|
||||
|
||||
class _ControlChannel(BaseCallback):
|
||||
"""把 :class:`_ControlPoll` 接到 SB3 的 step 回调上。
|
||||
|
||||
为什么不用信号：Windows 没有 SIGSTOP，且 ``os.kill(SIGTERM)`` 会退化成
|
||||
``TerminateProcess`` 硬杀，收尾分支根本不会执行。控制文件在 POSIX 与
|
||||
Windows 上都只是普通文件 IO，语义完全一致。
|
||||
|
||||
暂停是**协作式**的：在 ``_on_step`` 内阻塞轮询，训练因此停在 step 边界，
|
||||
进程仍存活、显存仍占用，但不消耗算力；收到 stop 立刻转入收尾落盘。
|
||||
"""
|
||||
|
||||
def __init__(self, path: Path, *, poll_every: int = 32,
|
||||
poll_s: float = 0.2):
|
||||
super().__init__()
|
||||
self.poller = _ControlPoll(path, poll_every=poll_every,
|
||||
poll_s=poll_s)
|
||||
|
||||
def paused_seconds(self) -> float:
|
||||
return self.poller.paused_total_s
|
||||
|
||||
def _event(self, name: str) -> None:
|
||||
print(json.dumps({"event": name,
|
||||
"timesteps": int(self.model.num_timesteps)}),
|
||||
flush=True)
|
||||
|
||||
def _on_step(self) -> bool:
|
||||
state = self.poller.poll()
|
||||
if state.stop:
|
||||
_request_stop("control_file")
|
||||
return False
|
||||
if state.paused:
|
||||
stopped = self.poller.wait_while_paused(
|
||||
lambda: self._event("paused"),
|
||||
lambda: self._event("resumed"))
|
||||
if stopped:
|
||||
_request_stop("control_file")
|
||||
return False
|
||||
return not stop_state["requested"]
|
||||
return True
|
||||
|
||||
control_path = getattr(args, "control_file", None)
|
||||
control_cb = _ControlChannel(Path(control_path)) if control_path else None
|
||||
|
||||
class _GracefulStop(BaseCallback):
|
||||
"""收到停止请求后在最近一次 step 边界结束 rollout，让模型能落盘。"""
|
||||
|
||||
def _on_step(self) -> bool:
|
||||
if stop_state["requested"]:
|
||||
print(json.dumps({"event": "graceful_stop",
|
||||
"signal": stop_state["signal"],
|
||||
"timesteps": int(self.model.num_timesteps)}),
|
||||
flush=True)
|
||||
return False
|
||||
return True
|
||||
|
||||
class _WallClockStop(BaseCallback):
|
||||
"""墙钟硬上限；暂停时间不计入，避免"挂起一小时回来就判超时"。"""
|
||||
|
||||
def __init__(self, max_wall_s: float, paused_clock=None):
|
||||
super().__init__()
|
||||
self.max_wall_s = max_wall_s
|
||||
self.start = time.monotonic()
|
||||
self.paused_clock = paused_clock
|
||||
|
||||
def paused_seconds(self) -> float:
|
||||
return float(self.paused_clock()) if self.paused_clock else 0.0
|
||||
|
||||
def _on_step(self) -> bool:
|
||||
elapsed = time.monotonic() - self.start - self.paused_seconds()
|
||||
if elapsed >= self.max_wall_s:
|
||||
print(json.dumps({"event": "wall_clock_stop",
|
||||
"elapsed_s": elapsed}))
|
||||
return False
|
||||
return True
|
||||
|
||||
class _ProgressJSON(BaseCallback):
|
||||
"""向训练前端输出单行 JSON 进度；不影响 SB3 自身 logger。
|
||||
|
||||
前端首个 curve 点越早出现越好，因此开头几步按更小间隔发送（前 8 个
|
||||
进度点逐步放大到 ``every_steps``），之后稳定在 ``every_steps``。
|
||||
这样短程 smoke 也能看到完整曲线，长训不会刷屏。
|
||||
"""
|
||||
|
||||
def __init__(self, every_steps: int):
|
||||
super().__init__()
|
||||
self.every_steps = max(1, int(every_steps))
|
||||
# init/resume 时计数器从上一阶段继续，前端要用增量算进度
|
||||
self.start = int(model.num_timesteps)
|
||||
self.last_emit = self.start
|
||||
self.emitted = 0
|
||||
|
||||
def _next_gap(self) -> int:
|
||||
warmup = max(0.2, 1.0 - self.emitted / 8.0)
|
||||
return max(1, min(self.every_steps, int(round(self.every_steps * warmup))))
|
||||
|
||||
def _on_step(self) -> bool:
|
||||
current = int(self.model.num_timesteps)
|
||||
if current - self.last_emit < self._next_gap():
|
||||
return True
|
||||
self.last_emit = current
|
||||
self.emitted += 1
|
||||
payload = _progress_payload(self.model, self.start)
|
||||
print(json.dumps(payload, ensure_ascii=False, allow_nan=False),
|
||||
flush=True)
|
||||
return True
|
||||
|
||||
started = time.monotonic()
|
||||
checkpoints_dir = (Path(args.checkpoint_dir) if args.checkpoint_dir
|
||||
else run_dir / "checkpoints")
|
||||
checkpoint_stem = f"{model_path.stem}_ckpt"
|
||||
|
||||
class _CheckpointJSON(BaseCallback):
|
||||
"""可续训落盘：滚动恢复点 + 周期 checkpoint + resume.json 指针。
|
||||
|
||||
- **滚动恢复点** ``<stem>_latest.zip``：每 ``recovery_every`` 个环境步
|
||||
原子覆盖写一次，训练开始前也先写一份。它保证"急停"（SIGKILL /
|
||||
``taskkill /F``，没有任何 Python 清理机会）之后**最多只丢
|
||||
``recovery_every`` 个环境步**，而且文件始终完整可加载。
|
||||
- **周期 checkpoint** ``<stem>_<步数>.zip``：每 ``every_steps`` 步留一份，
|
||||
只保留最近 ``keep`` 个，用于返祖对比与多档恢复。
|
||||
- 每次落盘都刷新 ``<run_dir>/resume.json``：无论怎么停，都有一份权威的
|
||||
"从哪继续"指针（含增量语义与生效过的 PPO 超参）。
|
||||
"""
|
||||
|
||||
def __init__(self, every_steps: int, keep: int, recovery_every: int):
|
||||
super().__init__()
|
||||
self.every_steps = max(0, int(every_steps))
|
||||
self.keep = max(1, int(keep))
|
||||
self.recovery_every = max(0, int(recovery_every))
|
||||
self.last_saved = int(model.num_timesteps)
|
||||
self.last_recovery = int(model.num_timesteps)
|
||||
self.saved: list[str] = []
|
||||
self.latest_path = checkpoints_dir / f"{checkpoint_stem}_latest.zip"
|
||||
|
||||
def write_recovery(self, reason: str) -> None:
|
||||
"""立刻写一份滚动恢复点；训练开始前调用即"起步基线"。
|
||||
|
||||
用闭包里的 ``model`` 而不是 ``self.model``：``BaseCallback.model``
|
||||
要等 ``learn()`` 时才被赋值，而起步基线恰恰在 ``learn()`` 之前写。
|
||||
"""
|
||||
current = int(model.num_timesteps)
|
||||
self.last_recovery = current
|
||||
self._write_recovery(current, reason)
|
||||
|
||||
def _on_step(self) -> bool:
|
||||
current = int(model.num_timesteps)
|
||||
if (self.recovery_every > 0
|
||||
and current - self.last_recovery >= self.recovery_every):
|
||||
self.last_recovery = current
|
||||
self._write_recovery(current, "interval")
|
||||
if (self.every_steps > 0
|
||||
and current - self.last_saved >= self.every_steps):
|
||||
self.last_saved = current
|
||||
self._write_interval(current)
|
||||
return True
|
||||
|
||||
def _persist(self, target: Path, current: int, reason: str,
|
||||
extra: dict[str, Any]) -> None:
|
||||
"""原子落盘 zip + 元数据，并刷新 resume.json 指针。"""
|
||||
_save_model_atomic(model, target)
|
||||
_write_metadata(
|
||||
model_path=target, args=args,
|
||||
wall_s=time.monotonic() - started, n_envs=args.envs,
|
||||
actual_timesteps=current, monitor_paths=monitor_paths,
|
||||
model=model,
|
||||
extra={"checkpoint_timesteps": current,
|
||||
"checkpoint_reason": reason,
|
||||
"checkpoint_of": str(model_path), **extra})
|
||||
_write_resume_hint(run_dir=run_dir, model_path=target,
|
||||
timesteps=current, reason=reason,
|
||||
model=model)
|
||||
|
||||
def _write_recovery(self, current: int, reason: str) -> None:
|
||||
checkpoints_dir.mkdir(parents=True, exist_ok=True)
|
||||
self._persist(self.latest_path, current, reason,
|
||||
{"recovery": True})
|
||||
print(json.dumps({"event": "recovery_checkpoint",
|
||||
"reason": reason, "timesteps": current,
|
||||
"model": str(self.latest_path)},
|
||||
ensure_ascii=False), flush=True)
|
||||
|
||||
def _write_interval(self, current: int) -> None:
|
||||
checkpoints_dir.mkdir(parents=True, exist_ok=True)
|
||||
target = checkpoints_dir / f"{checkpoint_stem}_{current:09d}.zip"
|
||||
self._persist(target, current, "interval", {"checkpoint": True})
|
||||
removed = _prune_checkpoints(checkpoints_dir, checkpoint_stem,
|
||||
self.keep)
|
||||
self.saved.append(str(target))
|
||||
print(json.dumps({"event": "checkpoint", "timesteps": current,
|
||||
"model": str(target), "pruned": removed},
|
||||
ensure_ascii=False), flush=True)
|
||||
|
||||
progress_every = (args.progress_every if args.progress_every > 0
|
||||
else args.rollout_steps * args.envs)
|
||||
checkpoint_cb = _CheckpointJSON(args.checkpoint_every, args.checkpoint_keep,
|
||||
args.recovery_every)
|
||||
if args.recovery_every > 0:
|
||||
# 起步基线：训练还没跑就急停，也至少有一份可加载、可续训的产物
|
||||
checkpoint_cb.write_recovery("baseline")
|
||||
# 顺序有语义：控制通道最先（暂停/停止要在本步立刻生效），进度次之，
|
||||
# 墙钟上限要能读到控制通道累计的暂停秒数，checkpoint 放最后。
|
||||
callbacks: list[Any] = []
|
||||
if control_cb is not None:
|
||||
callbacks.append(control_cb)
|
||||
if args.progress_json:
|
||||
callbacks.append(_ProgressJSON(progress_every))
|
||||
callbacks.append(_GracefulStop())
|
||||
callbacks.append(_WallClockStop(
|
||||
args.max_wall_s,
|
||||
paused_clock=(control_cb.paused_seconds if control_cb is not None
|
||||
else None)))
|
||||
callbacks.append(checkpoint_cb)
|
||||
|
||||
actual_timesteps = int(model.num_timesteps)
|
||||
try:
|
||||
model.learn(total_timesteps=args.timesteps,
|
||||
callback=callbacks if len(callbacks) > 1 else callbacks[0],
|
||||
progress_bar=False,
|
||||
reset_num_timesteps=not (args.resume or args.init))
|
||||
except BaseException as exc: # 训练失败也必须留下可续训产物，不静默
|
||||
failure = {
|
||||
"event": "training_failed",
|
||||
"error": f"{type(exc).__name__}: {exc}",
|
||||
"model": str(model_path),
|
||||
"timesteps": int(model.num_timesteps),
|
||||
}
|
||||
emergency = run_dir / f"{model_path.stem}_emergency.zip"
|
||||
try:
|
||||
_save_model_atomic(model, emergency)
|
||||
_write_metadata(model_path=emergency, args=args,
|
||||
wall_s=time.monotonic() - started,
|
||||
n_envs=args.envs,
|
||||
actual_timesteps=int(model.num_timesteps),
|
||||
monitor_paths=monitor_paths, model=model,
|
||||
extra={"emergency": True,
|
||||
"error": failure["error"]})
|
||||
_write_resume_hint(run_dir=run_dir, model_path=emergency,
|
||||
timesteps=int(model.num_timesteps),
|
||||
reason="emergency", model=model)
|
||||
failure["emergency_model"] = str(emergency)
|
||||
except Exception as save_exc: # 落盘失败不能再掩盖原始异常
|
||||
failure["emergency_save_error"] = (
|
||||
f"{type(save_exc).__name__}: {save_exc}")
|
||||
_write_json_atomic(run_dir / "training_failure.json", failure)
|
||||
print(json.dumps(failure, ensure_ascii=False), file=sys.stderr,
|
||||
flush=True)
|
||||
raise
|
||||
wall_s = time.monotonic() - started
|
||||
actual_timesteps = int(model.num_timesteps)
|
||||
stopped_by_signal = bool(stop_state["requested"])
|
||||
_save_model_atomic(model, model_path)
|
||||
metadata = _write_metadata(
|
||||
model_path=model_path, args=args, wall_s=wall_s, n_envs=args.envs,
|
||||
actual_timesteps=actual_timesteps, monitor_paths=monitor_paths,
|
||||
model=model,
|
||||
extra={"stopped_by_signal": stopped_by_signal,
|
||||
"stop_signal": stop_state["signal"],
|
||||
"checkpoint_files": checkpoint_cb.saved,
|
||||
"recovery_model": str(checkpoint_cb.latest_path)})
|
||||
# 最终模型也刷新一次 resume.json：优雅收尾后"从哪继续"指向最终模型
|
||||
_write_resume_hint(run_dir=run_dir, model_path=model_path,
|
||||
timesteps=actual_timesteps,
|
||||
reason=("stopped" if stopped_by_signal else "finished"),
|
||||
model=model)
|
||||
metadata_path = model_path.with_suffix(".json")
|
||||
vec_env.close()
|
||||
# 收尾阶段恢复默认处理器：再次 Ctrl+C / 停止可立即强杀，不再等待评估。
|
||||
for _signum, _handler in previous_handlers.items():
|
||||
try:
|
||||
signal.signal(_signum, _handler)
|
||||
except (ValueError, OSError):
|
||||
pass
|
||||
if stopped_by_signal:
|
||||
print(json.dumps({"event": "saved_after_stop",
|
||||
"model": str(model_path),
|
||||
"timesteps": actual_timesteps}), flush=True)
|
||||
|
||||
evaluation = _evaluate_env(
|
||||
model, problem=args.problem, stage_index=args.stage,
|
||||
episodes=args.eval_episodes, seed=args.seed + 1_000_000,
|
||||
profile=args.config_profile, reward_profile=args.reward_profile)
|
||||
manifest_eval = None
|
||||
if args.eval_manifest:
|
||||
try:
|
||||
manifest_eval = _evaluate_manifest(
|
||||
model_path, args.eval_manifest, limit=args.eval_manifest_limit)
|
||||
except Exception as exc:
|
||||
manifest_eval = {"error": f"{type(exc).__name__}: {exc}"}
|
||||
metadata["evaluation"] = evaluation
|
||||
metadata["eval_manifest"] = manifest_eval
|
||||
metadata_path.write_text(json.dumps(metadata, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8")
|
||||
print(json.dumps(metadata, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--problem", type=int, choices=(3, 4), required=True)
|
||||
parser.add_argument("--stage", type=int, choices=(1, 2, 3, 4),
|
||||
default=4)
|
||||
parser.add_argument("--timesteps", type=int, required=True)
|
||||
parser.add_argument("--seed", type=int, default=320001)
|
||||
parser.add_argument("--envs", type=int, default=1)
|
||||
parser.add_argument("--vec-env", choices=("dummy", "subproc"), default="dummy")
|
||||
parser.add_argument("--device", choices=("cpu", "cuda", "mps", "auto"),
|
||||
default="auto")
|
||||
parser.add_argument("--config-profile",
|
||||
choices=("production", "coarse-pretrain"),
|
||||
default="production")
|
||||
parser.add_argument("--model", required=True)
|
||||
parser.add_argument("--resume", action="store_true",
|
||||
help="从 --model 续训（沿用 checkpoint 内的 PPO 超参）")
|
||||
parser.add_argument("--init", default=None,
|
||||
help="可选：Q3→Q4 或继续预训练权重路径")
|
||||
parser.add_argument("--force", action="store_true")
|
||||
parser.add_argument("--eval-manifest", default=None)
|
||||
parser.add_argument("--eval-manifest-limit", type=int, default=0)
|
||||
parser.add_argument("--eval-episodes", type=int, default=5)
|
||||
parser.add_argument("--local-selector", choices=("geometry", "rl"),
|
||||
default="geometry")
|
||||
parser.add_argument("--local-model", default=None)
|
||||
parser.add_argument("--rollout-steps", type=int, default=256)
|
||||
parser.add_argument("--batch-size", type=int, default=256)
|
||||
parser.add_argument("--n-epochs", type=int, default=10)
|
||||
parser.add_argument("--max-wall-s", type=float, default=1200.0,
|
||||
help="训练墙钟硬上限；默认 20 分钟")
|
||||
parser.add_argument("--reward-profile", choices=("time", "prompt"),
|
||||
default="time",
|
||||
help="time=P0.4 纯时间口径；prompt=原始 §5.2 塑形对照")
|
||||
parser.add_argument("--progress-json", action="store_true",
|
||||
help="向训练前端输出单行 JSON 进度（UI 自动启用）")
|
||||
parser.add_argument("--control-file", default=None,
|
||||
help="训练前端控制文件（JSON：paused/stop）；"
|
||||
"跨平台暂停与优雅停止的主通道")
|
||||
parser.add_argument("--progress-every", type=int, default=0,
|
||||
help="JSON 进度间隔环境步数；0=每个 rollout")
|
||||
parser.add_argument("--checkpoint-every", type=int, default=20000,
|
||||
help="每 N 个环境步留一个历史 checkpoint；0=关闭（默认 20000）")
|
||||
parser.add_argument("--recovery-every", type=int, default=2000,
|
||||
help="每 N 个环境步原子覆盖写一份滚动恢复点 "
|
||||
"<模型目录>/checkpoints/<stem>_latest.zip；"
|
||||
"急停后最多丢 N 步。0=关闭（默认 2000）")
|
||||
parser.add_argument("--checkpoint-keep", type=int, default=3,
|
||||
help="checkpoint 保留最近 K 个（默认 3）")
|
||||
parser.add_argument("--checkpoint-dir", default=None,
|
||||
help="checkpoint 目录；默认 <模型目录>/checkpoints")
|
||||
parser.add_argument("--verbose", type=int, default=1)
|
||||
args = parser.parse_args(argv)
|
||||
if args.timesteps <= 0 or args.envs <= 0:
|
||||
parser.error("timesteps 与 envs 必须为正")
|
||||
if args.envs * args.rollout_steps < 2:
|
||||
parser.error("PPO rollout 至少需要 2 个样本")
|
||||
if args.eval_episodes < 0:
|
||||
parser.error("eval-episodes 不能为负")
|
||||
if args.checkpoint_every < 0:
|
||||
parser.error("checkpoint-every 不能为负")
|
||||
if args.recovery_every < 0:
|
||||
parser.error("recovery-every 不能为负")
|
||||
if args.checkpoint_keep < 1:
|
||||
parser.error("checkpoint-keep 至少为 1")
|
||||
return run_training(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,266 +0,0 @@
|
||||
"""低层 PPO 训练：8 方向到达任务（BC 直线专家热启动 + PPO 微调）。
|
||||
|
||||
产物：
|
||||
output/hrl/hrl_low.zip / hrl_low.json PPO 模型（部署用）
|
||||
output/hrl/bc_low_expert.npz / bc_low.pt 行为克隆数据与网络
|
||||
output/hrl/hrl_low_curve_env*.monitor.csv 每个环境独立学习曲线
|
||||
|
||||
评估口径：到达成功率 + 路径效率（直线距离/实际路径长，1 为完美直线）。
|
||||
这是独立导航实验：目前低层 PPO 仅供轨迹诊断，不影响正式 HTTP 任务。
|
||||
|
||||
示例：
|
||||
uv run python scripts/train_hrl_low.py --timesteps 60000
|
||||
uv run python scripts/train_hrl_low.py --skip-bc # 纯 PPO 对照
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from mcm26b.config import ProblemConfig
|
||||
from mcm26b.hrl.constants import (FEATURE_VERSION, LOW_MAX_STEPS,
|
||||
LOW_NUM_ACTIONS, LOW_OBS_DIM)
|
||||
from mcm26b.hrl.low_level_env import LowLevelEnv
|
||||
|
||||
HIDDEN = (64, 64)
|
||||
|
||||
|
||||
def _bc_module(obs_dim: int, n_actions: int, hidden: tuple[int, ...]):
|
||||
import torch
|
||||
|
||||
class BCMlp(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
layers = []
|
||||
prev = obs_dim
|
||||
for width in hidden:
|
||||
layers += [torch.nn.Linear(prev, width), torch.nn.Tanh()]
|
||||
prev = width
|
||||
layers.append(torch.nn.Linear(prev, n_actions))
|
||||
self.net = torch.nn.Sequential(*layers)
|
||||
|
||||
def forward(self, x):
|
||||
return self.net(x)
|
||||
|
||||
return BCMlp
|
||||
|
||||
|
||||
def collect(episodes: int, seed: int, config: ProblemConfig):
|
||||
import numpy as np
|
||||
env = LowLevelEnv(config=config, seed=seed)
|
||||
observations, actions, episode_ids = [], [], []
|
||||
for episode in range(episodes):
|
||||
obs, _info = env.reset(seed=seed + episode)
|
||||
for _step in range(LOW_MAX_STEPS + 2):
|
||||
observations.append(obs)
|
||||
actions.append(env.expert_action())
|
||||
episode_ids.append(episode)
|
||||
obs, _r, terminated, truncated, _info = env.step(env.expert_action())
|
||||
if terminated or truncated:
|
||||
break
|
||||
env.close()
|
||||
return (np.asarray(observations, dtype=np.float32),
|
||||
np.asarray(actions, dtype=np.int64),
|
||||
np.asarray(episode_ids, dtype=np.int64))
|
||||
|
||||
|
||||
def bc_validation_mask(episode_ids, *, seed: int):
|
||||
"""按整局留出，避免同一轨迹相邻状态跨训练/验证集。"""
|
||||
import numpy as np
|
||||
unique = np.unique(episode_ids)
|
||||
if len(unique) < 2:
|
||||
raise ValueError("BC 至少需要 2 局才能按整局划分训练/验证集")
|
||||
rng = np.random.default_rng(seed)
|
||||
validation_ids = rng.permutation(unique)[:max(1, len(unique) // 10)]
|
||||
return np.isin(episode_ids, validation_ids)
|
||||
|
||||
|
||||
def train_bc(x, y, episode_ids, *, epochs: int, batch_size: int, lr: float,
|
||||
device: str, seed: int):
|
||||
import torch
|
||||
BCMlp = _bc_module(LOW_OBS_DIM, LOW_NUM_ACTIONS, HIDDEN)
|
||||
model = BCMlp().to(device)
|
||||
optimizer = torch.optim.AdamW(model.parameters(), lr=lr)
|
||||
valid = torch.as_tensor(
|
||||
bc_validation_mask(episode_ids, seed=seed), device=device)
|
||||
x_t = torch.as_tensor(x, device=device)
|
||||
y_t = torch.as_tensor(y, device=device)
|
||||
x_v, y_v, x_tr, y_tr = x_t[valid], y_t[valid], x_t[~valid], y_t[~valid]
|
||||
loss_fn = torch.nn.CrossEntropyLoss()
|
||||
for epoch in range(epochs):
|
||||
order = torch.randperm(len(x_tr), device=device)
|
||||
total = correct = 0.0
|
||||
for start in range(0, len(order), batch_size):
|
||||
idx = order[start:start + batch_size]
|
||||
logits = model(x_tr[idx])
|
||||
loss = loss_fn(logits, y_tr[idx])
|
||||
optimizer.zero_grad()
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
total += float(loss.item()) * len(idx)
|
||||
correct += int((logits.argmax(1) == y_tr[idx]).sum().item())
|
||||
if epoch == 0 or (epoch + 1) % 5 == 0:
|
||||
with torch.no_grad():
|
||||
valid_acc = float((model(x_v).argmax(1) == y_v).float().mean().item())
|
||||
print(json.dumps({"event": "bc_epoch", "epoch": epoch + 1,
|
||||
"loss": round(total / len(x_tr), 4),
|
||||
"train_acc": round(correct / len(x_tr), 4),
|
||||
"valid_acc": round(valid_acc, 4)},
|
||||
ensure_ascii=False))
|
||||
with torch.no_grad():
|
||||
valid_acc = float((model(x_v).argmax(1) == y_v).float().mean().item())
|
||||
return model, valid_acc
|
||||
|
||||
|
||||
def copy_into_ppo(bc_model, device: str, env, *, seed: int,
|
||||
n_steps: int = 512) -> object:
|
||||
"""把 BC 权重拷入 MaskablePPO（policy_net 两层 + action_net），前向校验。"""
|
||||
import torch
|
||||
from sb3_contrib import MaskablePPO
|
||||
ppo = MaskablePPO("MlpPolicy", env, policy_kwargs={"net_arch": list(HIDDEN)},
|
||||
n_steps=n_steps, batch_size=128, n_epochs=6,
|
||||
gamma=1.0, gae_lambda=0.95, learning_rate=3e-4,
|
||||
clip_range=0.2, ent_coef=0.005, vf_coef=0.5,
|
||||
max_grad_norm=0.5, seed=seed, device=device)
|
||||
state = {key.removeprefix("net."): value
|
||||
for key, value in bc_model.state_dict().items()}
|
||||
policy_state = {key: value for key, value in state.items()
|
||||
if key in ("0.weight", "0.bias", "2.weight", "2.bias")}
|
||||
ppo.policy.mlp_extractor.policy_net.load_state_dict(policy_state)
|
||||
ppo.policy.action_net.load_state_dict(
|
||||
{"weight": state["4.weight"], "bias": state["4.bias"]})
|
||||
probe = torch.randn((8, LOW_OBS_DIM), device=device)
|
||||
with torch.no_grad():
|
||||
copied = ppo.policy.action_net(ppo.policy.mlp_extractor.policy_net(probe))
|
||||
if not torch.allclose(bc_model(probe), copied, atol=1e-6):
|
||||
raise RuntimeError("行为克隆与 PPO 前向映射不一致")
|
||||
return ppo
|
||||
|
||||
|
||||
def evaluate(model, *, episodes: int, seed: int,
|
||||
config: ProblemConfig) -> dict:
|
||||
"""确定性策略评估：成功率与路径效率（对照直线）。"""
|
||||
env = LowLevelEnv(config=config, seed=seed)
|
||||
successes = 0
|
||||
efficiencies = []
|
||||
for episode in range(episodes):
|
||||
obs, info = env.reset(seed=seed + episode)
|
||||
start, goal = tuple(info["position"]), tuple(info["goal"])
|
||||
straight = math.dist(start, goal)
|
||||
steps = 0
|
||||
for _ in range(LOW_MAX_STEPS + 2):
|
||||
action, _state = model.predict(obs, deterministic=True)
|
||||
obs, _r, terminated, truncated, info = env.step(int(action))
|
||||
steps += 1
|
||||
if terminated or truncated:
|
||||
break
|
||||
successes += int(terminated)
|
||||
efficiencies.append(min(1.0, straight /
|
||||
max(info["path_length_m"], 1e-9)))
|
||||
env.close()
|
||||
return {"episodes": episodes,
|
||||
"success_rate": successes / episodes,
|
||||
"mean_path_efficiency": sum(efficiencies) / len(efficiencies)}
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--timesteps", type=int, default=60_000)
|
||||
parser.add_argument("--seed", type=int, default=20260911)
|
||||
parser.add_argument("--bc-episodes", type=int, default=600)
|
||||
parser.add_argument("--bc-epochs", type=int, default=12)
|
||||
parser.add_argument("--envs", type=int, default=4)
|
||||
parser.add_argument("--rollout-steps", type=int, default=512,
|
||||
help="每环境每次 PPO rollout 长度；短 smoke 可设 8")
|
||||
parser.add_argument("--device", default="cpu")
|
||||
parser.add_argument("--model", default="output/hrl/hrl_low.zip")
|
||||
parser.add_argument("--eval-episodes", type=int, default=40)
|
||||
parser.add_argument("--skip-bc", action="store_true",
|
||||
help="跳过行为克隆（纯 PPO 对照实验用）")
|
||||
parser.add_argument("--verbose", type=int, default=1)
|
||||
args = parser.parse_args(argv)
|
||||
if min(args.envs, args.timesteps, args.eval_episodes,
|
||||
args.rollout_steps) <= 0:
|
||||
parser.error("envs/timesteps/eval-episodes/rollout-steps 必须为正数")
|
||||
if args.envs * args.rollout_steps < 2:
|
||||
parser.error("PPO rollout 至少需要 2 个样本")
|
||||
if not args.skip_bc and (args.bc_episodes < 2 or args.bc_epochs < 1):
|
||||
parser.error("BC 至少需要 2 局和 1 个 epoch")
|
||||
|
||||
import torch
|
||||
torch.set_num_threads(1)
|
||||
from sb3_contrib import MaskablePPO
|
||||
from stable_baselines3.common.monitor import Monitor
|
||||
from stable_baselines3.common.vec_env import DummyVecEnv
|
||||
from stable_baselines3.common.utils import set_random_seed
|
||||
set_random_seed(args.seed)
|
||||
|
||||
model_path = Path(args.model)
|
||||
model_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
curve_paths = [model_path.with_name(
|
||||
f"{model_path.stem}_curve_env{rank}.monitor.csv")
|
||||
for rank in range(args.envs)]
|
||||
|
||||
def _factory(rank: int):
|
||||
def _make():
|
||||
env = Monitor(LowLevelEnv(seed=args.seed + rank),
|
||||
filename=str(curve_paths[rank]))
|
||||
return env
|
||||
return _make
|
||||
|
||||
vec_env = DummyVecEnv([_factory(rank) for rank in range(args.envs)])
|
||||
started = time.monotonic()
|
||||
bc_valid_acc = None
|
||||
if args.skip_bc:
|
||||
model = MaskablePPO("MlpPolicy", vec_env,
|
||||
policy_kwargs={"net_arch": list(HIDDEN)},
|
||||
n_steps=args.rollout_steps, batch_size=128, n_epochs=6,
|
||||
gamma=1.0, gae_lambda=0.95, learning_rate=3e-4,
|
||||
clip_range=0.2, ent_coef=0.005, vf_coef=0.5,
|
||||
max_grad_norm=0.5, seed=args.seed,
|
||||
device=args.device, verbose=args.verbose)
|
||||
else:
|
||||
x, y, episode_ids = collect(args.bc_episodes, args.seed, ProblemConfig())
|
||||
bc_model, bc_valid_acc = train_bc(
|
||||
x, y, episode_ids, epochs=args.bc_epochs, batch_size=256,
|
||||
lr=3e-4, device=args.device, seed=args.seed)
|
||||
model = copy_into_ppo(bc_model, args.device, vec_env, seed=args.seed,
|
||||
n_steps=args.rollout_steps)
|
||||
bc_dir = model_path.parent / "bc_low.pt"
|
||||
torch.save(bc_model.state_dict(), str(bc_dir))
|
||||
print(json.dumps({"event": "bc_done", "samples": int(len(x))},
|
||||
ensure_ascii=False))
|
||||
model.learn(total_timesteps=args.timesteps,
|
||||
progress_bar=args.verbose > 0)
|
||||
elapsed_s = time.monotonic() - started
|
||||
|
||||
model.save(str(model_path))
|
||||
eval_seed = args.seed + 1_000_000
|
||||
evaluation = evaluate(model, episodes=args.eval_episodes, seed=eval_seed,
|
||||
config=ProblemConfig())
|
||||
report = {
|
||||
"level": "low", "timesteps": args.timesteps, "seed": args.seed,
|
||||
"device": args.device, "wall_seconds": round(elapsed_s, 1),
|
||||
"model": str(model_path), "algorithm": "maskable_ppo",
|
||||
"feature_version": FEATURE_VERSION, "bc_skipped": args.skip_bc,
|
||||
"bc_valid_acc": bc_valid_acc, "evaluation_seed": eval_seed,
|
||||
"monitor_files": [str(path) for path in curve_paths],
|
||||
"training_role": "standalone_navigation_only",
|
||||
"obs_dim": LOW_OBS_DIM, "act_dim": LOW_NUM_ACTIONS,
|
||||
"model_sha256": hashlib.sha256(model_path.read_bytes()).hexdigest(),
|
||||
**evaluation,
|
||||
}
|
||||
model_path.with_suffix(".json").write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
vec_env.close()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,5 +0,0 @@
|
||||
"""启动 HRL 模型训练前端（薄入口，便于加入支撑材料命令）。"""
|
||||
from mcm26b.training_ui.server import run_training_ui
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(run_training_ui())
|
||||
@@ -1,195 +0,0 @@
|
||||
"""问题3/4 强化学习选点策略训练（PPO + CUDA）。
|
||||
|
||||
训练环境 mcm26b.rl.env.LocalizationEnv 与运行闭环共享 features 观测/候选
|
||||
布局。模型输出到 output/rl/q3_ppo.zip（问题3）或 q4_ppo.zip（问题4），
|
||||
由 `mcm26b run --strategy rl` / `--q4-strategy rl` 加载。
|
||||
|
||||
示例：
|
||||
uv sync --group rl --group dev
|
||||
uv run python scripts/train_rl.py --problem 3 --timesteps 30000
|
||||
uv run python scripts/train_rl.py --problem 4 --timesteps 30000
|
||||
uv run mcm26b run --robot-id 2026001 --self-test --strategy rl
|
||||
uv run mcm26b run --robot-id 2026001 --self-test --problem 4 --q4-strategy rl
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
from dataclasses import asdict
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
from stable_baselines3 import PPO
|
||||
from sb3_contrib import MaskablePPO
|
||||
from stable_baselines3.common.vec_env import DummyVecEnv
|
||||
|
||||
from mcm26b.config import ProblemConfig
|
||||
from mcm26b.rl.env import LocalizationEnv
|
||||
|
||||
|
||||
def _make_env(problem: int, seed: int, config: ProblemConfig,
|
||||
directional_probability: float):
|
||||
def _factory():
|
||||
env = LocalizationEnv(config, problem=problem,
|
||||
directional_probability=directional_probability,
|
||||
real_budget_s=3600.0)
|
||||
env.reset(seed=seed)
|
||||
return env
|
||||
return _factory
|
||||
|
||||
|
||||
def evaluate(model, problem: int, *, episodes: int = 24,
|
||||
directional_probability: float = 0.5,
|
||||
config: ProblemConfig = ProblemConfig()) -> dict:
|
||||
"""独立场景评估：确定性策略的成功率、步数与平均回报。"""
|
||||
env = LocalizationEnv(config, problem=problem,
|
||||
directional_probability=directional_probability)
|
||||
total_reward = 0.0
|
||||
successes = 0
|
||||
steps = []
|
||||
costs = []
|
||||
for episode in range(episodes):
|
||||
obs, _info = env.reset(seed=1_000_000 + episode)
|
||||
episode_reward = 0.0
|
||||
for step in range(env.max_episode_steps + 2):
|
||||
kwargs = {'action_masks': env.action_masks()} if isinstance(model, MaskablePPO) else {}
|
||||
action, _state = model.predict(obs, deterministic=True, **kwargs)
|
||||
obs, reward, terminated, truncated, info = env.step(int(action))
|
||||
episode_reward += float(reward)
|
||||
if terminated or truncated:
|
||||
break
|
||||
total_reward += episode_reward
|
||||
cleared = bool(info.get("cleared"))
|
||||
successes += int(cleared)
|
||||
steps.append(step + 1)
|
||||
costs.append(info['virtual_cost_s'])
|
||||
env.close()
|
||||
return {
|
||||
"episodes": episodes,
|
||||
"success_rate": successes / episodes,
|
||||
"mean_steps": sum(steps) / episodes,
|
||||
"mean_episode_reward": total_reward / episodes,
|
||||
"successes": successes,
|
||||
"mean_virtual_cost_s": sum(costs) / episodes,
|
||||
"max_virtual_cost_s": max(costs),
|
||||
}
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--problem", type=int, choices=(3, 4), default=3,
|
||||
help="训练问题：3=全向源选点，4=含定向遮挡梯子")
|
||||
parser.add_argument("--timesteps", type=int, default=30000,
|
||||
help="PPO 总环境步数")
|
||||
parser.add_argument("--seed", type=int, default=20260911)
|
||||
parser.add_argument("--envs", type=int, default=2, help="并行环境份数（CPU 侧）")
|
||||
parser.add_argument("--directional-probability", type=float, default=None,
|
||||
help="问题4训练场景中定向源比例（默认 1.0，先学遮挡分支）")
|
||||
parser.add_argument("--device", default="auto",
|
||||
help="auto=优先 CUDA；或 cuda/cpu")
|
||||
parser.add_argument("--model", default=None,
|
||||
help="模型输出路径（默认 output/rl/q<problem>_ppo.zip）")
|
||||
parser.add_argument("--init", default=None,
|
||||
help="初始模型路径（如 output/rl/q3_ppo_bc_init.zip；"
|
||||
"加载后继续 PPO 微调）")
|
||||
parser.add_argument("--eval-episodes", type=int, default=24)
|
||||
parser.add_argument("--verbose", type=int, default=1)
|
||||
parser.add_argument('--algorithm', choices=('ppo', 'maskable_ppo'), default='maskable_ppo')
|
||||
parser.add_argument('--torch-threads', type=int, default=1)
|
||||
args = parser.parse_args(argv)
|
||||
directional_probability = (args.directional_probability
|
||||
if args.directional_probability is not None
|
||||
else (1.0 if args.problem == 4 else 0.0))
|
||||
if not 0.0 <= directional_probability <= 1.0:
|
||||
parser.error("--directional-probability 须在 [0,1]")
|
||||
|
||||
import torch
|
||||
torch.set_num_threads(args.torch_threads)
|
||||
if args.device == "auto":
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
else:
|
||||
device = args.device
|
||||
if device == "cuda" and not torch.cuda.is_available():
|
||||
parser.error("--device cuda 但当前环境检测不到 CUDA GPU")
|
||||
print(json.dumps({
|
||||
"event": "train_start", "problem": args.problem,
|
||||
"timesteps": args.timesteps, "device": device,
|
||||
"directional_probability": directional_probability,
|
||||
"torch": torch.__version__,
|
||||
"gpu": torch.cuda.get_device_name(0) if device == "cuda" else None,
|
||||
"cuda_available": torch.cuda.is_available(),
|
||||
}, ensure_ascii=False))
|
||||
|
||||
config = ProblemConfig()
|
||||
model_cls = MaskablePPO if args.algorithm == 'maskable_ppo' else PPO
|
||||
vec_env = DummyVecEnv(
|
||||
[_make_env(args.problem, args.seed + rank, config,
|
||||
directional_probability)
|
||||
for rank in range(max(1, args.envs))])
|
||||
if args.init:
|
||||
metadata = json.loads(Path(args.init).with_suffix('.json').read_text(encoding='utf-8'))
|
||||
if (metadata.get('algorithm', 'ppo') != args.algorithm
|
||||
or metadata.get('feature_version') != 2):
|
||||
parser.error('初始化模型的算法/特征版本与当前训练不符，请重训或提供v2初始化模型')
|
||||
model = model_cls.load(args.init, env=vec_env, device=device)
|
||||
else:
|
||||
model = model_cls(
|
||||
"MlpPolicy", vec_env,
|
||||
policy_kwargs={"net_arch": [128, 128]},
|
||||
n_steps=512,
|
||||
batch_size=128,
|
||||
n_epochs=6,
|
||||
gamma=1.0,
|
||||
gae_lambda=0.95,
|
||||
learning_rate=3e-4,
|
||||
clip_range=0.2,
|
||||
ent_coef=0.005,
|
||||
vf_coef=0.5,
|
||||
max_grad_norm=0.5,
|
||||
seed=args.seed,
|
||||
device=device,
|
||||
verbose=args.verbose,
|
||||
)
|
||||
|
||||
started = time.monotonic()
|
||||
model.learn(total_timesteps=args.timesteps,
|
||||
progress_bar=args.verbose > 0)
|
||||
elapsed_s = time.monotonic() - started
|
||||
|
||||
model_path = Path(args.model) if args.model \
|
||||
else Path("output/rl") / f"q{args.problem}_ppo.zip"
|
||||
model_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
model.save(str(model_path))
|
||||
|
||||
evaluation = evaluate(model, args.problem, episodes=args.eval_episodes,
|
||||
directional_probability=directional_probability,
|
||||
config=config)
|
||||
report = {
|
||||
"problem": args.problem,
|
||||
"timesteps": args.timesteps,
|
||||
"seed": args.seed,
|
||||
"device": device,
|
||||
"directional_probability": directional_probability,
|
||||
"wall_seconds": round(elapsed_s, 1),
|
||||
"model": str(model_path),
|
||||
"init": args.init,
|
||||
"algorithm": args.algorithm,
|
||||
"feature_version": 2,
|
||||
"reward": 'negative_virtual_cost_div100_success20_failure100',
|
||||
"gamma": 1.0,
|
||||
"config": asdict(config),
|
||||
"model_sha256": hashlib.sha256(model_path.read_bytes()).hexdigest(),
|
||||
**evaluation,
|
||||
}
|
||||
report_path = model_path.with_suffix(".json")
|
||||
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
vec_env.close()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,27 +0,0 @@
|
||||
"""HRL Optuna TPE 调参入口（P2 待实现）。
|
||||
|
||||
搜索空间按用户 Prompt §10，但只允许在不破坏覆盖证明/证书/预算的工程参数上搜索：
|
||||
粒子数、重采样阈值、信息增益权重、移动成本权重、检测成本权重、候选点距离、
|
||||
覆盖网格间距、停止连续无信号次数、清除距离阈值。目标 = 平均定位清除时间 +
|
||||
未清除惩罚，每个配置至少 5 个随机种子。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--problem", type=int, choices=(3, 4), default=3)
|
||||
parser.add_argument("--trials", type=int, default=30)
|
||||
parser.add_argument("--timesteps", type=int, default=2048)
|
||||
parser.add_argument("--manifest", default="output/hrl/manifests/dev.json")
|
||||
parser.add_argument("--output", default="output/hrl/p2-tuning")
|
||||
parser.parse_args(argv)
|
||||
raise NotImplementedError(
|
||||
"P2 尚未实现：调参必须建立在 P1 smoke 与稳定基准之上，"
|
||||
"且不能搜索真实误差上限或破坏覆盖证明的网格参数。")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,123 +0,0 @@
|
||||
"""Offline audit of official RL practice runs (Q3/Q4); never connects to simulator.
|
||||
|
||||
Checks the copied official .result.json/.jlog, our session.jsonl and summary.json:
|
||||
package sha256, enter/exit, accepted timestamps inside official window, unique
|
||||
request ids, clear-success channel count == official jammer_count, summary
|
||||
consistency, and virtual time recomputation with integer microsecond accumulation.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def save_json(path: Path, value: dict) -> None:
|
||||
path.write_text(json.dumps(value, ensure_ascii=False, indent=2,
|
||||
allow_nan=False) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def audit(folder: Path) -> dict:
|
||||
records = [json.loads(line) for line in
|
||||
(folder / "session.jsonl").read_text(encoding="utf-8").splitlines()]
|
||||
events = [e for e in records if e.get("event") == "accepted"]
|
||||
summary = json.loads((folder / "summary.json").read_text(encoding="utf-8"))
|
||||
official_paths = list(folder.glob("*.result.json"))
|
||||
if len(official_paths) != 1 or not events:
|
||||
raise ValueError("Expected one official result and nonempty session evidence")
|
||||
official = json.loads(official_paths[0].read_text(encoding="utf-8"))
|
||||
package = official_paths[0].with_name(
|
||||
official_paths[0].name.replace(".result.json", ".jlog"))
|
||||
checks: dict = {}
|
||||
|
||||
checks["official_problem_matches_summary"] = (
|
||||
official["problem_no"] in (3, 4)
|
||||
and summary["strategy"] == ("rl" if official["problem_no"] == 3 else "rl_q4")
|
||||
and 10 <= official["jammer_count"] <= 16
|
||||
and official["omnidirectional_jammer_count"] + official["directional_jammer_count"]
|
||||
== official["jammer_count"])
|
||||
checks["official_package_sha256_matches"] = (
|
||||
hashlib.sha256(package.read_bytes()).hexdigest() == official["package_sha256"])
|
||||
checks["normal_enter_and_exit"] = (
|
||||
events[0]["path"] == "/enter" and events[-1]["path"] == "/exit"
|
||||
and events[-1]["response"].get("exit_reason") == "user_exit")
|
||||
start_ms = datetime.fromisoformat(official["window_started_at_utc"]).timestamp() * 1000
|
||||
end_ms = datetime.fromisoformat(official["ended_at_utc"]).timestamp() * 1000
|
||||
checks["responses_within_official_run_window"] = all(
|
||||
start_ms <= e["response"]["real_timestamp_ms"] <= end_ms + 1 for e in events)
|
||||
ids = [e["request"]["request_id"] for e in events]
|
||||
checks["unique_accepted_request_ids"] = len(ids) == len(set(ids))
|
||||
clears = [e for e in events if e["path"] == "/clear"]
|
||||
successes = [e for e in clears if e["response"].get("clear_result") == "success"]
|
||||
cleared = sorted({e["request"]["channel"] for e in successes})
|
||||
checks["all_official_sources_cleared"] = len(cleared) == official["jammer_count"]
|
||||
checks["summary_completed_and_consistent"] = (
|
||||
bool(summary["completed"])
|
||||
and sorted(summary["cleared_channels"]) == cleared
|
||||
and summary["unresolved_channels"] == []
|
||||
and summary["anomalies"] == [])
|
||||
|
||||
position, channel, virtual_us = (0.0, 0.0), 1, 0
|
||||
mismatches = 0
|
||||
for event in events:
|
||||
request, response, path = event["request"], event["response"], event["path"]
|
||||
distance, switches, action_s = 0.0, 0, 0
|
||||
if path in ("/measure", "/clear"):
|
||||
target = (request["position"]["x"], request["position"]["y"])
|
||||
distance = math.dist(position, target)
|
||||
position = target
|
||||
if path == "/measure":
|
||||
switches = int(channel != request["channel"])
|
||||
channel = request["channel"]
|
||||
action_s = 5
|
||||
else:
|
||||
action_s = 5 if response["clear_result"] == "success" else 3
|
||||
virtual_us += round(distance / 5 * 1_000_000) + (switches + action_s) * 1_000_000
|
||||
if round(response["virtual_time_s"] * 1_000_000) != virtual_us:
|
||||
mismatches += 1
|
||||
checks["all_timestamps_match_microsecond_recomputation"] = mismatches == 0
|
||||
real_s = (events[-1]["response"]["real_timestamp_ms"]
|
||||
- events[0]["response"]["real_timestamp_ms"]) / 1000
|
||||
checks["within_real_and_virtual_limits"] = (
|
||||
real_s < events[0]["response"]["remaining_real_duration_s"]
|
||||
and virtual_us / 1_000_000 < events[0]["response"]["max_virtual_duration_s"])
|
||||
|
||||
report = {
|
||||
"passed": all(checks.values()),
|
||||
"source": "official_practice",
|
||||
"strategy": summary["strategy"],
|
||||
"problem": official["problem_no"],
|
||||
"case_code": official["case_code"],
|
||||
"official_jammer_count": official["jammer_count"],
|
||||
"directional_jammer_count": official["directional_jammer_count"],
|
||||
"cleared_channels": cleared,
|
||||
"accepted_requests_including_enter_exit": len(events),
|
||||
"measures": len([e for e in events if e["path"] == "/measure"]),
|
||||
"clear_attempts": len(clears),
|
||||
"clear_successes": len(successes),
|
||||
"virtual_time_s": virtual_us / 1_000_000,
|
||||
"real_elapsed_server_s": real_s,
|
||||
"checks": checks,
|
||||
"limits": "仅验证本官方演练案例；不触碰正式测试，不解读官方加密日志内容。",
|
||||
}
|
||||
save_json(folder / "verification.json", report)
|
||||
manifest = {p.name: {"bytes": p.stat().st_size,
|
||||
"sha256": hashlib.sha256(p.read_bytes()).hexdigest()}
|
||||
for p in sorted(folder.iterdir()) if p.is_file()
|
||||
and p.name != "manifest.json"}
|
||||
save_json(folder / "manifest.json", manifest)
|
||||
return report
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("folder", type=Path)
|
||||
args = parser.parse_args()
|
||||
result = audit(args.folder)
|
||||
print(json.dumps({"passed": result["passed"], "case_code": result["case_code"],
|
||||
"problem": result["problem"], "checks": result["checks"]},
|
||||
ensure_ascii=False, indent=2))
|
||||
raise SystemExit(0 if result["passed"] else 1)
|
||||
Reference in New Issue
Block a user