From 61fd22598a91d704cb334c1b6a7310d474bcf665 Mon Sep 17 00:00:00 2001 From: Horldsence Date: Sat, 12 Sep 2026 08:47:51 +0800 Subject: [PATCH] =?UTF-8?q?revert:=20=E7=A7=BB=E9=99=A4=E5=85=A8=E9=83=A8?= =?UTF-8?q?=E6=9C=BA=E5=99=A8=E5=AD=A6=E4=B9=A0=E4=BB=A3=E7=A0=81=EF=BC=8C?= =?UTF-8?q?=E5=9B=9E=E9=80=80=E5=88=B0=E6=97=A0=20ML=20=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 删除 RL/HRL/训练前端及其文档、脚本、测试与依赖： - src/mcm26b/{rl,hrl,training_ui}/、planner/{rl,hrl}_loop.py、process_control.py - project/（PPO 策略、训练、环境、评估） - scripts/{train_rl,train_hrl_*,benchmark_rl,benchmark_hrl,pretrain_*_expert 等} - tests/{test_rl*,test_hrl*,test_training_ui*}、tests/js/training_ui_harness.cjs - docs/{algo-rl,hrl-*,training-ui*} 与相关计划条目、论文 PDF 回退 CLI 恢复为 grid/seven/bayes + directional，移除 --strategy rl|hrl、 --q4-strategy、--rl-model、--hrl-* 等参数；pyproject 删除 rl 依赖组与 PyTorch CUDA index，uv.lock 同步重生成。 保留此后的非 ML 改动（referee 回放、Q1/Q3 脚本、代码清理等）。 验证：pytest 232 passed；Q3/Q4 --self-test 全清通过。 --- ...ss_cf390556-4ef3-4ce6-b91e-782aee1718f9.md | 87 -- README.md | 28 +- docs/B题建模分析与实施计划.md | 1 - docs/algo-rl.md | 167 --- docs/hrl-p1-execution-report.md | 117 --- docs/hrl-sol-execution.md | 267 ----- docs/paper/main.pdf | Bin 413577 -> 413578 bytes docs/training-ui-windows-audit.md | 261 ----- docs/training-ui.md | 160 --- docs/强化学习适用性审核与改进-20260911.md | 165 --- logs/README.md | 4 - models/README.md | 4 - project/README.md | 29 - project/__init__.py | 4 - project/baselines/__init__.py | 3 - project/baselines/end_to_end_ppo.py | 16 - project/baselines/heuristic.py | 16 - project/env/__init__.py | 6 - project/env/local_env.py | 22 - project/env/particle_filter.py | 4 - project/env/simulator_client.py | 8 - project/eval/__init__.py | 4 - project/eval/ablation.py | 20 - project/eval/evaluate.py | 17 - project/policies/__init__.py | 5 - project/policies/high_level_ppo.py | 10 - project/policies/low_level_ppo.py | 9 - project/train/__init__.py | 5 - project/train/curriculum.py | 6 - project/train/domain_randomization.py | 31 - project/train/optuna_tune.py | 20 - pyproject.toml | 19 +- scripts/ablate_hrl.py | 25 - scripts/benchmark_hrl.py | 428 -------- scripts/benchmark_rl.py | 128 --- scripts/evaluate_rl_revision.py | 125 --- scripts/pretrain_hrl_expert.py | 24 - scripts/pretrain_rl_expert.py | 255 ----- scripts/save_official_rl_summary.py | 36 - scripts/summarize_rl_revision.py | 78 -- scripts/train_hrl_high.py | 987 ------------------ scripts/train_hrl_low.py | 266 ----- scripts/train_hrl_ui.py | 5 - scripts/train_rl.py | 195 ---- scripts/tune_hrl.py | 27 - scripts/verify_rl_practice.py | 123 --- src/mcm26b/cli.py | 69 +- src/mcm26b/hrl/__init__.py | 14 - src/mcm26b/hrl/constants.py | 49 - src/mcm26b/hrl/curriculum.py | 109 -- src/mcm26b/hrl/executor.py | 113 -- src/mcm26b/hrl/hierarchical_env.py | 260 ----- src/mcm26b/hrl/low_level_env.py | 158 --- src/mcm26b/hrl/policy.py | 135 --- src/mcm26b/hrl/scheduler.py | 495 --------- src/mcm26b/planner/hrl_loop.py | 172 --- src/mcm26b/planner/rl_loop.py | 94 -- src/mcm26b/process_control.py | 175 ---- src/mcm26b/rl/__init__.py | 14 - src/mcm26b/rl/_constants.py | 11 - src/mcm26b/rl/env.py | 331 ------ src/mcm26b/rl/features.py | 209 ---- src/mcm26b/rl/policy.py | 95 -- src/mcm26b/training_ui/__init__.py | 15 - src/mcm26b/training_ui/__main__.py | 4 - src/mcm26b/training_ui/analysis.py | 397 ------- src/mcm26b/training_ui/demo.py | 223 ---- src/mcm26b/training_ui/schema.py | 328 ------ src/mcm26b/training_ui/server.py | 977 ----------------- src/mcm26b/training_ui/static/app.js | 841 --------------- src/mcm26b/training_ui/static/charts.js | 255 ----- src/mcm26b/training_ui/static/index.html | 180 ---- src/mcm26b/training_ui/static/style.css | 230 ---- tests/js/training_ui_harness.cjs | 285 ----- tests/test_hrl.py | 286 ----- tests/test_hrl_checkpoint.py | 444 -------- tests/test_hrl_low_revision.py | 119 --- tests/test_hrl_p1.py | 86 -- tests/test_hrl_scheduler_revision.py | 88 -- tests/test_rl.py | 101 -- tests/test_rl_revision.py | 97 -- tests/test_training_ui.py | 957 ----------------- tests/test_training_ui_js.py | 44 - uv.lock | 491 --------- 84 files changed, 9 insertions(+), 13159 deletions(-) delete mode 100644 .zcode/plans/plan-sess_cf390556-4ef3-4ce6-b91e-782aee1718f9.md delete mode 100644 docs/algo-rl.md delete mode 100644 docs/hrl-p1-execution-report.md delete mode 100644 docs/hrl-sol-execution.md delete mode 100644 docs/training-ui-windows-audit.md delete mode 100644 docs/training-ui.md delete mode 100644 docs/强化学习适用性审核与改进-20260911.md delete mode 100644 logs/README.md delete mode 100644 models/README.md delete mode 100644 project/README.md delete mode 100644 project/__init__.py delete mode 100644 project/baselines/__init__.py delete mode 100644 project/baselines/end_to_end_ppo.py delete mode 100644 project/baselines/heuristic.py delete mode 100644 project/env/__init__.py delete mode 100644 project/env/local_env.py delete mode 100644 project/env/particle_filter.py delete mode 100644 project/env/simulator_client.py delete mode 100644 project/eval/__init__.py delete mode 100644 project/eval/ablation.py delete mode 100644 project/eval/evaluate.py delete mode 100644 project/policies/__init__.py delete mode 100644 project/policies/high_level_ppo.py delete mode 100644 project/policies/low_level_ppo.py delete mode 100644 project/train/__init__.py delete mode 100644 project/train/curriculum.py delete mode 100644 project/train/domain_randomization.py delete mode 100644 project/train/optuna_tune.py delete mode 100644 scripts/ablate_hrl.py delete mode 100644 scripts/benchmark_hrl.py delete mode 100644 scripts/benchmark_rl.py delete mode 100644 scripts/evaluate_rl_revision.py delete mode 100644 scripts/pretrain_hrl_expert.py delete mode 100644 scripts/pretrain_rl_expert.py delete mode 100644 scripts/save_official_rl_summary.py delete mode 100644 scripts/summarize_rl_revision.py delete mode 100644 scripts/train_hrl_high.py delete mode 100644 scripts/train_hrl_low.py delete mode 100644 scripts/train_hrl_ui.py delete mode 100644 scripts/train_rl.py delete mode 100644 scripts/tune_hrl.py delete mode 100644 scripts/verify_rl_practice.py delete mode 100644 src/mcm26b/hrl/__init__.py delete mode 100644 src/mcm26b/hrl/constants.py delete mode 100644 src/mcm26b/hrl/curriculum.py delete mode 100644 src/mcm26b/hrl/executor.py delete mode 100644 src/mcm26b/hrl/hierarchical_env.py delete mode 100644 src/mcm26b/hrl/low_level_env.py delete mode 100644 src/mcm26b/hrl/policy.py delete mode 100644 src/mcm26b/hrl/scheduler.py delete mode 100644 src/mcm26b/planner/hrl_loop.py delete mode 100644 src/mcm26b/planner/rl_loop.py delete mode 100644 src/mcm26b/process_control.py delete mode 100644 src/mcm26b/rl/__init__.py delete mode 100644 src/mcm26b/rl/_constants.py delete mode 100644 src/mcm26b/rl/env.py delete mode 100644 src/mcm26b/rl/features.py delete mode 100644 src/mcm26b/rl/policy.py delete mode 100644 src/mcm26b/training_ui/__init__.py delete mode 100644 src/mcm26b/training_ui/__main__.py delete mode 100644 src/mcm26b/training_ui/analysis.py delete mode 100644 src/mcm26b/training_ui/demo.py delete mode 100644 src/mcm26b/training_ui/schema.py delete mode 100644 src/mcm26b/training_ui/server.py delete mode 100644 src/mcm26b/training_ui/static/app.js delete mode 100644 src/mcm26b/training_ui/static/charts.js delete mode 100644 src/mcm26b/training_ui/static/index.html delete mode 100644 src/mcm26b/training_ui/static/style.css delete mode 100644 tests/js/training_ui_harness.cjs delete mode 100644 tests/test_hrl.py delete mode 100644 tests/test_hrl_checkpoint.py delete mode 100644 tests/test_hrl_low_revision.py delete mode 100644 tests/test_hrl_p1.py delete mode 100644 tests/test_hrl_scheduler_revision.py delete mode 100644 tests/test_rl.py delete mode 100644 tests/test_rl_revision.py delete mode 100644 tests/test_training_ui.py delete mode 100644 tests/test_training_ui_js.py diff --git a/.zcode/plans/plan-sess_cf390556-4ef3-4ce6-b91e-782aee1718f9.md b/.zcode/plans/plan-sess_cf390556-4ef3-4ce6-b91e-782aee1718f9.md deleted file mode 100644 index f0e3c24..0000000 --- a/.zcode/plans/plan-sess_cf390556-4ef3-4ce6-b91e-782aee1718f9.md +++ /dev/null @@ -1,87 +0,0 @@ -# 分层强化学习（HRL）实施计划 — 2026 高教社杯 B 题第三、四问 - -## 0. 现状与复用决策 - -仓库已有生产级基建，**全部复用、不重写**： - -| 现有组件 | 位置 | 用途 | -|---|---|---| -| `SimulatorClient`（HTTP+JSON、幂等重试、截止时钟、JSONL 日志） | `src/mcm26b/protocol/` | 模拟器对接（prompt 第 1 步已完成） | -| 本地评测机（内存回环，一局秒级） | `referee/` + `harness.LocalSimTransport` | 本地简化环境内核 | -| 网格幸存集滤波（真源保留定理已证明） | `src/mcm26b/particles.py` | 每频道信念后验 | -| 凸候选集 + 五状态机 | `src/mcm26b/belief/` | 频道状态观测 | -| 信息增益选点、TSP 2-opt、六边形覆盖、清除证书、后备清除 | `src/mcm26b/planner/` | 候选生成器 + 证书审批兜底 + 启发式基线 | -| 单步 MaskablePPO 全套（特征/掩码/BC/训练/审计） | `src/mcm26b/rl/` + `scripts/` | 代码模式模板 + 端到端对照基线 | - -**改进方向**（来自 v2 审计文档已知局限）：单步 RL 只做选点、学不到全场调度；Q4 后备清除偏多、尾部退化（13778s→11797s，距 4000s 目标远）。HRL 高层正是补**跨频道全场调度**决策。 - -**已确认的三个决策**：① 代码集成到 `src/mcm26b/hrl/`（README 给出与 prompt 目录树的映射表）；② 低层为字面 8 方向 PPO + 直线控制器对照（诚实预期：部署时只提交终点，低层对虚拟时间增益≈0，价值在架构完整性与消融验证）；③ 本次跑通全流程 + 短程训练（1–2h），完整课程训练交付脚本由用户自行启动。 - -## 1. 新建文件 - -``` -src/mcm26b/hrl/ -├── __init__.py -├── constants.py # HRL 接口常量：N_CANDIDATES=96、观测维度，feature_version=3 -├── candidates.py # 高层候选生成：待清除源/信息增益探索点/覆盖验证点/Q4 扇区探测点 + 动作掩码 -├── features.py # 高层观测（20 频道×14 统计 + 全局标量 + 96×6 候选特征 ≈864 维）、低层观测 -├── policy.py # HighLevelPolicy/LowLevelPolicy 加载器（沿用 PointSelectionPolicy 元数据校验模式） -├── low_level_env.py # 低层训练环境：随机起点→子目标，Discrete(8)，逐步计费，奖励=−Δd−转向+到达 -├── hierarchical_env.py # 高层整场环境：10–16 源，动作=子目标槽位，内嵌低层执行器，域随机化+课程参数 -└── executor.py # 子目标执行器：低层 PPO 步进（本地计费）或直线控制器（可切换），终点提交模拟器 - -scripts/ -├── train_hrl_low.py # 低层：BC 克隆直线控制器热启动 → MaskablePPO 微调；直线对照评估 -├── train_hrl_high.py # 高层：4 阶段课程（--stage --resume --timesteps），SubprocVecEnv 并行 -├── pretrain_hrl_expert.py # 可选：从 BayesLoop 调度器克隆 (obs, 子目标) 对热启动高层 -├── tune_hrl.py # Optuna TPE（prompt §10 搜索空间，预算 15–20 trials，初值复用 bayes 已调参数） -├── benchmark_hrl.py # 配对消融：HRL vs bayes/directional/单步 PPO，dev(700-702)/val(91000-91005)/stress 固定场景 -└── ablate_hrl.py # 四项消融：无信息增益奖励/无动作掩码/无课程/无粒子滤波 - -src/mcm26b/planner/hrl_loop.py # HRLLoop 运行时闭环（Q3/Q4），保留证书审批与异常留痕 -src/mcm26b/cli.py # 新增 --strategy hrl / --q4-strategy hrl / --hrl-model -tests/test_hrl.py # 特征布局、掩码、环境形状、奖励恒等式、训练-部署观测一致性 -docs/hrl.md # 设计文档 + prompt 目录树映射表 -docs/paper/ 增补 # 算法流程图、伪代码、超参表、训练曲线 -``` - -## 2. 核心设计 - -**高层动作空间** Discrete(96) + 掩码：槽位按固定顺序填充（已定位待清除源 ≤16 → PF 驱动探索点 → 覆盖网格未验证点 → Q4 扇区探测点），空槽/已清除频道/接收盘外候选掩码 −∞。 - -**高层观测**（≈864 维，参照 v2 的 401 维布局模式）：20 频道 × 14 特征（六状态 one-hot、MEC μx/μy、σx/σy/ρ、log 幸存格数、期望信息增益、距离、频道号）+ 全局 8 维（位置、时间比例、已清除数、最近检测结果编码/示向度/距离）+ 96×6 候选特征。 - -**step 语义**：高层选子目标 → executor 步进（低层 8 方向，每步 ≤100m，本地按 Δd/5 计费；部署时仅终点调模拟器）→ 按子目标类型执行 /measure 或 /clear → belief+PF 更新 → 奖励。 - -**奖励**：`r = −Δ虚拟耗时/100 + 100·清除 + 5·新检测 + w_info·ΔH − 冗余惩罚`，主口径与目标函数（虚拟时间）一致；w_info 可消融。 - -**课程 4 阶段**（prompt §7.1）：1 源/5 频道/无噪 → 3 源/10 频道/±0.5° → 8 源/16 频道/±1° → 10–16 源/20 频道/±1°；每阶段迁移上阶段权重，Q4 从 Q3 权重微调（定向比例递增），域随机化贯穿（源位置/频道/r_eff/朝向/噪声种子）。 - -**运行时闭环 HRLLoop**：初始覆盖相位（快速发现信号）→ 主循环高层决策（证书达标自动 /clear，与 BayesLoop 同语义）→ 全清即 /exit。不触碰候选集更新与清除判定的守恒性（真源保留定理依赖），异常留痕兜底保留。 - -## 3. 实施顺序（对应 prompt §12） - -1. `constants.py` + `candidates.py` + `features.py`（候选/掩码/观测）+ 单元测试 -2. `low_level_env.py` + `train_hrl_low.py`：低层短程训练，验证收敛到近直线（报告与直线最优的差距） -3. `hierarchical_env.py` + `executor.py` + `hierarchical_env` 单元测试（奖励恒等式：回合回报 = f(虚拟时间、清除数)） -4. `train_hrl_high.py` + 课程 + 域随机化 + SubprocVecEnv；`pretrain_hrl_expert.py` BC 热启动 -5. 短程课程训练（每阶段 2–5 万步，总计 1–2h），产出训练曲线 -6. `hrl_loop.py` + CLI 接入，`--local-judge` 本地演练验证全清正确性 -7. `benchmark_hrl.py` 配对对比（沿用 benchmark_rl.py 的 all_valid 有效口径）+ `ablate_hrl.py` 四项消融 -8. `tune_hrl.py` 受限 Optuna；产出最佳超参表 -9. 本地 referee 演练 ×3/题，表 1 格式结果（案例编码/清除数/平均定位清除时间/程序运行时间），日志 JSONL 原样归档 `output/hrl/` -10. `docs/hrl.md`、README 更新、论文素材（流程图/伪代码/超参表/对比分析，含"为什么分层 PPO、为什么非端到端、为什么非多智能体"论证） - -## 4. 验收标准 - -- 全部 pytest 通过（现有 243 项 + 新增 HRL 测试） -- `benchmark_hrl.py` 输出 all_valid=true（真值全清、缺席无误判、零异常、正常 exit、计时复算一致） -- 短程训练下 Q4 HRL 相对 directional 有可测改进方向（对标单步 RL 的 −14.38%）；Q3 如实报告与 bayes 的差距（v2 审计口径：Q3 RL 无稳定增益） -- 所有实验固定种子、命令可复现；消融表含均值±方差（≥5 种子） -- 完整训练只需用户执行 `scripts/train_hrl_high.py --full` 级别的命令 - -## 5. 风险与对策 - -- **高层整场回合过长（100–200 决策步）不收敛** → γ=1 + 密集 Δcost 奖励 + BC 从 bayes 调度器热启动 -- **低层增益≈0** → 如实消融报告（直线对照差异≈0），论文定位为架构必要性验证而非性能来源 -- **Q3 可能无增益** → 与 v2 口径一致如实报告，主打 Q4 调度收益；两种策略可共存（HRL 不改默认策略，仅在胜出后由用户决定切换默认） diff --git a/README.md b/README.md index 7c939fd..4d4badd 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,6 @@ - [协议规则表、字段表与状态机（P0 交付）](docs/协议规则表与状态机.md) - [问题一完整解答](docs/问题一解答.md) - [问题三搜索与总耗时改进方案（含七点覆盖证明和自建消融）](docs/algo-3.md) -- [问题3/4 强化学习选点（PPO + CUDA）](docs/algo-rl.md) -- [强化学习审核、修复与独立验证（2026-09-11）](docs/强化学习适用性审核与改进-20260911.md) -- [HRL 优化方案与 Sol 执行指南（含当前缺口、命令和验收门槛）](docs/hrl-sol-execution.md) -- [HRL P0+P1 执行报告（本轮实际命令、smoke 与多轮本地测试）](docs/hrl-p1-execution-report.md) - [论文](docs/paper/main.tex) ## 项目结构 @@ -25,13 +21,10 @@ src/mcm26b/ ├── belief/ # P3 逐频道候选集合 K_c：保守外包多边形 + 半平面裁剪（已完工） ├── coverage/ # P4 49点覆盖账本（网格生成 + 逐频道记录） ├── planner/ # P3 保证接收盘选点器 / P4 全向闭环 / P5 混合源定向闭环（已完工） -│ # P6b RL 接入：planner/rl_loop.py（PPO 选点，惰性加载 torch） -├── rl/ # P6b 强化学习扩展：训练环境、观测/候选特征、PPO 模型加载（可选依赖） ├── evaluation/ # 日志复算与统计（已完工） └── harness/ # 测试侧适配：评测机<->Transport 内存回环、故障注入、回放、场景 tests/ # pytest（含解耦守卫：运行时零导入 referee + 规则表不漂移） -scripts/ # 采集、基准、可视化与校核脚本（含问题四批量仿真 benchmark_q4.py、 - # RL 训练 train_rl.py 与 RL 配对消融 benchmark_rl.py） +scripts/ # 采集、基准、可视化与校核脚本（含问题四批量仿真 benchmark_q4.py） output/ examples/ # 演练证据、算例与生成产物 ``` @@ -71,29 +64,11 @@ Python ≥ 3.10，运行时依赖仅 `requests`；测试/自检另需 `pytest` ```bash uv sync # 安装（可编辑挂载本包与 referee） -uv sync --group rl --group dev # 追加 RL 依赖：torch(CUDA)+stable-baselines3+gymnasium uv run pytest # 全部测试（含 referee 90 项 + 解耦守卫） uv run python -m referee # 起本地评测机（可选，配合 WebUI 回放） uv run mcm26b sim-check # P0 自检：复现附件计时示例 105/111/194/199 uv run mcm26b run --robot-id 2026001 --self-test # 问题3自检：随机全向场景闭环（不连官方） uv run mcm26b run --robot-id 2026001 --self-test --problem 4 # 问题4自检：混合定向源场景 -# RL 策略（需先安装 rl 依赖组并训练模型；详见 docs/algo-rl.md）： -uv run python scripts/train_rl.py --problem 3 --timesteps 120000 # PPO + CUDA 训练 Q3 选点 -uv run python scripts/pretrain_rl_expert.py --problem 4 --episodes 3000 # 自建模拟器专家行为克隆 -uv run python scripts/train_rl.py --problem 4 --timesteps 40000 --init output/rl/q4_ppo_bc_init.zip -uv run mcm26b run --robot-id 2026001 --self-test --strategy rl # 问题3 RL 闭环 -uv run mcm26b run --robot-id 2026001 --self-test --problem 4 --q4-strategy rl # 问题4 RL 闭环 -uv run python scripts/benchmark_rl.py --seeds 5 # RL vs 手写策略配对消融 -uv run python scripts/verify_rl_practice.py output/rl/official-practice/q4/20260911-rl-bc-01 -uv run python scripts/benchmark_hrl.py --variants hrl_heuristic,bayes,directional \ - --manifest output/hrl/manifests/dev.json --output output/hrl/p1-heuristic -uv run python scripts/train_hrl_high.py --problem 3 --stage 1 --timesteps 2048 \ - --envs 1 --vec-env dummy --device mps --config-profile production \ - --reward-profile time --model output/hrl/p1-smoke/q3.zip --eval-episodes 2 -# 训练前端（本地回环 Web UI）：实时进程/进度/参数收敛曲线，详见 docs/training-ui.md -uv run python -m mcm26b.training_ui --ui-port 8036 # http://127.0.0.1:8036 -uv run mcm26b run --robot-id 2026001 --self-test --strategy hrl \ - --hrl-model output/hrl/p1-smoke/q3.zip --hrl-device mps # 自检默认：源数按题面 10~16 随机；实机预算 30 分钟（--real-budget-s 可改，仅自检生效）； # 找全后持续复扫未清除频道直到限时（--config 关闭 persistent_search 可回到"找全即退"） uv run mcm26b run --robot-id <队号> --log output/run/session.jsonl # 对接模拟器正式/演练 @@ -159,7 +134,6 @@ client.exit() | P4 问题3全向闭环（49点覆盖、定位队列、清除、停止） | ✅ 官方问题3演练单案例通过（10/10全清，[验证报告](output/q3/practice/20260911-port2026/report.md)） | | P5 问题4定向扩展（联合约束、后备清除、预算） | ✅ 自建完成（官方演练待执行） | | P6 加速与消融 | ✅ 七点批量（`--strategy seven`，官方演练 12/12）+ 贝叶斯闭环（`--strategy bayes` 默认，自建消融 −30.0%，[算法说明](docs/algo-3-bayes.md)） | -| P6b RL 选点 | ✅ PPO+CUDA 训练环境/闭环接入/官方+自建演练验证/专家 BC 初始化（Q3 `--strategy rl`，Q4 `--q4-strategy rl`，[算法说明](docs/algo-rl.md)） | ## 采集与正式测试约束 diff --git a/docs/B题建模分析与实施计划.md b/docs/B题建模分析与实施计划.md index c1857fd..2e90430 100644 --- a/docs/B题建模分析与实施计划.md +++ b/docs/B题建模分析与实施计划.md @@ -194,7 +194,6 @@ AI 的适合用法是把这些模块逐一变成可测试程序、生成边界 | P4 全向闭环 | 49点覆盖记录、定位队列、清除、停止、日志统计 | 10/16源、边界源和错误响应案例；官方演练全清并正常退出 | 自建闭环已完成：`planner.OmnidirectionalLoop`（10/16/0 源、近边界源、预算耗尽、故障注入测试通过）；官方演练全清待团队执行 | | P5 定向扩展 | no_signal联合约束、覆盖发现、有限清除后备与预算 | 朝外边界源不漏；真源不误删；格数与退出预算检查 | 已实现：`planner.DirectionalExtension`（49点覆盖对定向源成立、保证接收盘内 no_signal 记为观测到定向、对跖+环探梯子、`fallback_clear_cells` 有限圆盘覆盖后备清除带格数/预算双硬顶）；`tests/test_loop_q4.py` 全过；官方混合源演练待执行 | | P6 加速消融 | 共享测点、自适应覆盖、目标排序、提前清除 | 配对案例全清不退化，时间/请求数改善可复算 | 可删减；闭环基线数据已可用于消融对照 | -| P6b RL 选点 | 问题3/4 测量点选择的 PPO+CUDA 训练环境与闭环接入 | 自建全清；与手写策略配对消融有可复算报告 | 已完成：`scripts/train_rl.py`/`benchmark_rl.py`，模型 `output/rl/q{3,4}_ppo.zip`，说明 `docs/algo-rl.md` | | P7 正式与论文 | 固定版本、论文表1、日志原名、上传核查 | 问题3/4各3份正式记录与程序版本对应 | 团队后续安排；本任务不执行正式测试 | 按2026年9月10日起的剩余时间，建议10日补齐P1并衔接P3，11日完成P4自建边界案例与全向演练，12日完成P5及混合源演练；未达验收就删减P6并优先修正闭环。演练不限3次，正式机会为问题3和4各3次。团队在版本稳定后自行安排正式记录，留出失败恢复和论文整理时间。附件建议9月13日15:30前完成正式测试，17:30后不能启动新的演练或正式测试；已进入上传队列的日志仍会继续上传。此处引用所给附件安排，运行前核对模拟器界面公告；这份排期不改变当前禁止使用正式测试的限制。 diff --git a/docs/algo-rl.md b/docs/algo-rl.md deleted file mode 100644 index bddae47..0000000 --- a/docs/algo-rl.md +++ /dev/null @@ -1,167 +0,0 @@ -# 问题3/4 强化学习选点（PPO + CUDA） - -## 0. 摘要 - -**2026-09-11审核更新：**本文件下方的训练成绩与BC准确率是第一版历史记录。 -复核发现训练/部署特征错位、BC激活函数不一致以及评测只检查completed等问题。 -当前修复、Maskable PPO新模型和独立配对验证见 -[强化学习适用性审核与改进](强化学习适用性审核与改进-20260911.md)。 -新模型未替换原默认检查点；通过`--rl-model`显式加载。 - -本项目在保持 P0–P5 保守性与可解释证明不变的前提下，为问题3（全向源）与 -问题4（定向源）增加了一个可选的**强化学习测量点选择策略**： - -- 把“下一检测点去哪”建模为离散动作的局部决策问题；401维特征是压缩观测， - 不能认定为完整马尔可夫状态，尤其第四问仍有未知朝向和被省略的历史约束； -- 状态由逐频道信念的几何特征 + 60 个候选检测点的几何特征组成（401 维）； -- 动作为 60 选 1（保证接收盘环候选、对称交会候选、定向遮挡环探候选）； -- 使用 Stable-Baselines3 的 PPO 训练，MLP 策略与价值网络在 **CUDA GPU** - （本机 RTX 4060 Laptop, CUDA 12.8 / torch 2.11.0+cu128）上更新； -- 训练环境基于自建评测机 `referee`（内存回环，与真实 HTTP 协议同边界）； -- 运行闭环仅替换既有选点排序器，覆盖证明、清除证书、定向后备清除全部保留。 - -历史结果（第一版自建配对消融，5 个种子）：Q3 用 12 万步随机初始化 PPO（与 `bayes` -基本持平，+1.18%）；Q4 用**专家行为克隆初始化 + PPO 微调**，混合源平均 -虚拟时间相对 `directional` 基线降低 **7.16%**（14250.1s → 13230.2s）。 -Q3 的 BC 初始化在消融中反而退化，因此保留随机初始化模型；Q4 的 BC 初始化 -是当时五场比较中均值最好的版本。官方演练验证见 §5：两次 Q4 演练（3 定向混合源、10 定向全定向 -场景）与一次 Q3 演练均全清，且通过离线审计。 - -## 1. 环境搭建（CUDA） - -```powershell -# 安装可选依赖组 rl（PyTorch CUDA 12.8 + stable-baselines3 + gymnasium） -uv sync --group rl --group dev - -# 验证 CUDA -uv run python -c "import torch; print(torch.cuda.is_available(), torch.cuda.get_device_name(0))" -``` - -`pyproject.toml` 中 `[tool.uv.sources] torch = { index = "pytorch-cu128" }` -与 `[[tool.uv.index]]` 指向 PyTorch 官方 CUDA 12.8 轮子；下载约 2.6 GB， -首次同步建议 `$env:UV_HTTP_TIMEOUT='300'`。默认 PyPI 的 Windows torch 轮子 -为 `+cpu`，可以使用CPU训练；本轮修订模型即采用CPU，CUDA并非训练前提。 - -## 2. 第一版局部决策环境 - -训练环境：`src/mcm26b/rl/env.py::LocalizationEnv`。 - -| 项 | 定义 | -| --- | --- | -| 初始状态 | 自建模拟器随机布设一个源（Q3 全向；Q4 定向比例可调，训练默认 1.0），机器狗完成一次保证得到 `direction` 的首测 | -| 观测 | 401 维：21 个全局特征 + 20 维频道 one-hot + 60×6 维候选点特征（见 `features.py`） | -| 动作 | `Discrete(60)`：0=候选集 MEC 圆心，1..36=保证接收盘 3 环×12 方向，37..51=对称交会 15 点，52..59=定向遮挡环探 8 点 | -| 转移 | 按动作位置调用 `/measure`，用 `BeliefState` 保守更新候选集；Q4 保证接收盘内 `no_signal` 记为朝向遮挡并进入 `blocked` 序列；证书达标后自动 `/clear`；遮挡梯子用尽后模拟 20m 网格后备清除 | -| 奖励 | `-Δvirtual_time/100`（移动+检测+换频+清除真实耗时）、候选半径收缩加成、首次遮挡证据 +2、重复测点 -0.75、清除成功 +10、失败/截断负奖励 | - -约束说明：候选点全部来自既有几何生成器（`planner.candidate_points`、 -`planner.safe_disk_radius`、定向环探梯子），RL 不生成新的空间候选；它只做 -排序。清除仍须通过几何证书；但“保证收到方向”只适用于全向源且实际点位于 -距离保证盘内。第一版的52—59号环探点可能在盘外，第四问的定向遮挡也不受 -距离盘保证。v2对第三问加入盘外动作屏蔽，第四问保留探索分支。 - -## 3. 训练与自建模拟器逻辑利用 - -```powershell -# 1) 用自建模拟器上的手写专家生成 (观测, 动作) 并行为克隆 -uv run python scripts/pretrain_rl_expert.py --problem 3 --episodes 3000 -uv run python scripts/pretrain_rl_expert.py --problem 4 --episodes 3000 - -# 2) 从当前v2 BC初始化继续普通PPO（算法类型需显式匹配） -uv run python scripts/train_rl.py --problem 4 --algorithm ppo --timesteps 40000 \ - --init output/rl/q4_ppo_bc_init.zip - -# Q3 保留随机初始化长训练版本： -uv run python scripts/train_rl.py --problem 3 --timesteps 120000 -``` - -专家逻辑与自建模拟器同源：Q3 用 worst-posterior 选点器，Q4 用对跖/环探 -遮挡梯子；`pretrain_rl_expert.py` 在 `LocalizationEnv` 上记录每个状态的 -专家动作，训练与 PPO policy_net/action_net 同构的 MLP，再把权重拷入 PPO -作为初始化。产物位于 `output/rl/bc/` 与 `output/rl/q{3,4}_ppo_bc_init.zip`。 - -实测：Q3 专家数据 5141 条、BC 验证准确率 38.9%，微调后自建消融退化 -（保留 12 万步随机初始化模型）；Q4 专家数据 14181 条、BC 验证准确率 -65.3%，微调后自建消融从 −3.84% 提升到 **−7.16%**。但审核确认BC与PPO的 -激活函数不一致，这些数字不能证明成功迁移了专家策略，也不足以判断全向分支已最优。 - -输出：`output/rl/q3_ppo.zip`、`output/rl/q4_ppo.zip` 及同名 JSON 报告； -训练脚本先打印 `train_start`（含 device/gpu 字段），再执行 PPO 并在独立 -种子场景上确定性评估成功率、平均步数与平均回报。Q4 4 万步微调约 236s； -Q3 12 万步约 625s（RTX 4060 Laptop，2 个 CPU 环境）。 - -## 4. 运行闭环接入 - -- Q3：`uv run mcm26b run --robot-id 2026001 --self-test --strategy rl` - `RLBayesLoop` 继承 `BayesLoop`（七点覆盖 + 网格粒子滤波 + 滚动 2-opt）， - 仅 `_select_point` 改由 PPO 输出候选编号。 -- Q4：`uv run mcm26b run --robot-id 2026001 --self-test --problem 4 --q4-strategy rl` - `RLDirectionalLoop` 继承 `DirectionalExtension`（49 点覆盖 + 遮挡梯子 + - 20m 有限圆盘后备清除），仅 `_next_probe` 改由 PPO 输出。 -- 模型加载时校验观测 401 维 / 动作 60 维，失配直接拒绝运行。 -- 默认运行路径不导入 torch/referee；只有显式选择 `rl` 才惰性导入，项目 - 运行时依赖仍保持 `requests` 一项。 - -## 5. 消融基准 - -```powershell -uv run python scripts/benchmark_rl.py --seeds 5 -``` - -同种子同场景配对比较，结果写入 `output/rl/benchmark.json`。以下为第一版历史结果： - -| 场景 | 基线 | 平均虚拟时间（基线 / RL） | 相对差 | 全清 | -| --- | --- | --- | --- | --- | -| Q3（10 源全向 ×5 种子，scratch PPO） | `bayes` | 3088.3s / 3124.8s | +1.18% | 5/5 | -| Q4（10 源、3 定向 ×5 种子，BC+PPO） | `directional` | 14250.1s / 13230.2s | −7.16% | 5/5 | - -训练局评估：Q3 单源 40 局成功率 100%、平均 3.55 步；Q4 定向单源 40 局 -成功率 90%、平均 4.45 步；其余单源训练局未完成，不能由此宣称已兜底成功。随机策略对照： -Q3 平均回报 13.28 vs PPO 17.71；Q4 平均回报 8.72 vs BC+PPO 12.62。 - -### 5.1 官方模拟器演练验证（仅演练，未触碰正式测试） - -使用目录内官方 `jammers-simulator-full` 的**演练测试**（不限次数），通过 -官方 HTTP 接口运行同一闭环；官方产物按原名归档并离线审计： - -| 问题 | 案例 | 场景真值（演练结束后可见） | 结果 | 虚拟时间 | 请求数 | 审计 | -| --- | --- | --- | --- | --- | --- | --- | -| Q3 | `F38Y-9Q9J-B82V-CB9Z` | 13 全向 | 13/13 全清 | 3538.8s | 124 | ✅ | -| Q4 | `X43X-VMRF-8GRZ-3SYA` | 7 全向 + 3 定向 | 10/10 全清 | 14003.1s | 619 | ✅ | -| Q4(BC) | `BXSA-TFQ3-2DJ6-DHH7` | **10 定向** | 10/10 全清 | 14562.8s | 691 | ✅ | - -前两次为随机初始化模型，第三次为 BC 初始化微调模型。两个 Q4 官方案例的 -源构成不同（3 定向 vs 10 定向），**虚拟时间不可直接相减比较**；可确认的 -是 RL 在官方混合源与全定向源演练中都能完成全部清除，且 -`all_timestamps_match_microsecond_recomputation`、官方 `.jlog` 的 -SHA-256 校验全部通过。归档目录：`output/rl/official-practice/`。 - -### 5.2 自建 referee 演练验证 - -`uv run mcm26b run --local-judge` 走自建 referee 的 `q3_practice` / -`q4_practice` 模块（与官方相同的演练生命周期，真值在结束后可见）： - -| 问题 | 案例 | 场景 | 结果 | 虚拟时间 | 归档 | -| --- | --- | --- | --- | --- | --- | -| Q3 | `FKUY-K8KJ-RXRL-FMBM` | 10 全向 | 10/10 全清 | 3317.6s | `output/rl/local-practice/q3/20260911-rl-01` | -| Q4 | `3J8T-L9GY-HRGL-LYVZ` | 7 全向 + 3 定向 | 10/10 全清 | 11942.8s | `output/rl/local-practice/q4/20260911-rl-01` | -| Q4(BC) | `VM2G-UVC6-4N7R-TVGW` | 7 全向 + 3 定向 | 10/10 全清 | 12520.8s | `output/rl/local-practice/q4/20260911-rl-bc-01` | - -不同案例仍不可直接比较；成对可比的提升证据是 §5 的 5 种子同场景消融。 - -## 6. 结论与边界 - -以下是第一版结论；BC权重拷贝缺陷使“同构初始化”的解释不成立，性能判断 -应以新审核报告的同场景验证及不确定性为准，而不能由BC分类准确率推出。 - -- RL 已接入问题3/4 完整闭环，并在自建 referee 演练与官方模拟器演练中 - 全清验证；这是方法验证，不是对正式成绩的承诺。 -- 第一版Q4的BC初始化后均值变化为−3.84% → −7.16%，但初始化存在激活函数 - 缺陷，样本数也不足以支持“明确收益”的归因；当前判断见新审核报告。 -- 训练环境使用自建场景假设（源均匀分布、逐点固定误差场、定向源朝向随机）； - 官方隐藏分布未必相同。正式测试仍应默认使用经过演练验证的手写策略； - `--strategy rl` / `--q4-strategy rl` 适合消融、方法对比与后续调参。 -- 官方演练仅使用“演练测试”按钮与开放接口，从未点击“正式测试”；官方 - `.jlog/.result.json` 保持原样，未解密。 -- 保守性不变式：RL 只改变选点排序，不改变覆盖账本、候选集更新、清除证书 - 与后备清除的数学证明；所有策略均以 `/exit` 与完整日志收尾。 diff --git a/docs/hrl-p1-execution-report.md b/docs/hrl-p1-execution-report.md deleted file mode 100644 index 1be4011..0000000 --- a/docs/hrl-p1-execution-report.md +++ /dev/null @@ -1,117 +0,0 @@ -# HRL P0+P1 执行报告（本地自建评测机） - -日期：2026-09-11（本机工作区） -范围：按用户确认，仅完成 P0 正确性回归 + P1 最小可训练/可评测闭环； -P2 的 BC、完整课程长训、Optuna、正式消融与正式测试表留待下一阶段。 - -## 1. 本轮代码交付 - -| 文件 | 内容 | -| --- | --- | -| `scripts/benchmark_hrl.py` | 同一 manifest 真值场景复用；`hrl`/`hrl_heuristic`/`bayes`/`directional` 配对评测；写 `results.json`、`aggregate.json/csv`、每局 `summary.json` 与 `session.jsonl` | -| `scripts/train_hrl_high.py` | 高层 MaskablePPO 训练；`--problem/--stage/--envs/--vec-env/--device/--config-profile/--reward-profile/--model/--resume/--init/--eval-manifest`；模型元数据固化 feature/action/reward 版本；墙钟硬上限 | -| `src/mcm26b/hrl/constants.py` | 新增 `ACTION_LAYOUT_VERSION=1`、`REWARD_VERSION=1` | -| `src/mcm26b/hrl/policy.py` | 高层模型严格校验元数据；低层模型保持独立基础校验 | -| `src/mcm26b/cli.py` | 新增 `--strategy hrl`、`--q4-strategy hrl`、`--hrl-model`、`--hrl-low-model`、`--hrl-device`、`--trace-low` | -| `src/mcm26b/hrl/hierarchical_env.py` | 新增 `HRLReward.time_only()`：P0.4 建议的纯时间目标，去掉逐源清除/发现/熵减塑形 | -| `project/` | 严格目录树薄兼容层，转发真实实现，不复制算法 | -| `tests/test_hrl_p1.py` | manifest 往返、真值隔离、纯时间奖励、project re-export 回归 | - -## 2. 全量回归 - -```bash -.venv/bin/python -m pytest -q --tb=short -# 275 passed in 124.57s -``` - -## 3. P1 smoke 训练（本机 Apple MPS） - -```bash -.venv/bin/python scripts/train_hrl_high.py \ - --problem 3 --stage 1 --timesteps 2048 --seed 320001 \ - --envs 1 --vec-env dummy --device mps \ - --config-profile production --reward-profile time \ - --model output/hrl/p1-smoke-time/q3.zip \ - --eval-episodes 2 --rollout-steps 128 --batch-size 128 \ - --n-epochs 3 --max-wall-s 800 --verbose 0 -``` - -结果： - -- `actual_timesteps=2048`，墙钟 `260.069 s`； -- 保存后确定性评估 2/2 成功，`accounted=20`，无异常； -- 模型元数据含 `feature_version=4`、`action_layout_version=1`、 - `reward_version=1`、`reward_profile=time`、`model_sha256`。 - -模型重载已由以下两条路径验证；`--resume` 的 `--timesteps` 是增量步数 -（SB3 `reset_num_timesteps=False` 语义），元数据写 `timesteps_semantics="additional"`： - -```bash -.venv/bin/python -m mcm26b.cli run \ - --robot-id 2026001 --self-test --strategy hrl \ - --hrl-model output/hrl/p1-smoke-time/q3.zip --hrl-device mps \ - --sources 3 --seed 1 --summary-steps 5 - -python scripts/benchmark_hrl.py --variants hrl \ - --hrl-model output/hrl/p1-smoke-time/q3.zip --hrl-device mps \ - --problem 3 --scenes 1 --sources 3 --output output/hrl/p1-bench-hrl -``` - -## 4. 多轮本地配对测试 - -### Q3：3 个独立场景 × 3 个变体 - -```bash -python scripts/benchmark_hrl.py --variants hrl,hrl_heuristic,bayes \ - --hrl-model output/hrl/p1-smoke-time/q3.zip --hrl-device mps \ - --problem 3 --scenes 3 --sources 3 --start-seed 91000 \ - --output output/hrl/p1-multiround-q3 -``` - -| variant | episodes | valid_rate | cleared_mean | virtual_time_mean_s | virtual_time_std_s | program_run_time_mean_s | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| bayes | 3 | 1.0 | 3.0 | 2518.371 | 74.183 | 0.748 | -| hrl_heuristic | 3 | 1.0 | 3.0 | 2518.371 | 74.183 | 0.883 | -| hrl | 3 | 1.0 | 3.0 | 3731.380 | 120.536 | 1.853 | - -结论：P1 模型只在 stage 1 单源课程上训练 2048 步，可直接重载并在 3 源场景保持 3/3 有效全清； -但用时明显未优于 bayes/heuristic，符合本轮不做正式性能声明的边界。 - -### Q4：3 个独立混合源场景 × 2 个变体 - -```bash -python scripts/benchmark_hrl.py --variants hrl_heuristic,directional \ - --problem 4 --scenes 3 --sources 3 --directional 1 --start-seed 92000 \ - --output output/hrl/p1-multiround-q4 --force -``` - -| variant | episodes | valid_rate | cleared_mean | virtual_time_mean_s | virtual_time_std_s | program_run_time_mean_s | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| directional | 3 | 1.0 | 3.0 | 13442.997 | 263.902 | 1.063 | -| hrl_heuristic | 3 | 1.0 | 3.0 | 12292.198 | 344.827 | 6.284 | - -说明：Q4 高层 PPO 尚未训练；本轮只验证 Q4 的 `hrl_heuristic` 与 `directional` 闭环、真值隔离与计时审计。 - -## 5. P0 状态 - -- `--resume` 重载：16 步起步模型续训 32 增量步后 `actual_timesteps=48`， - 校验 `feature_version/action_layout_version/reward_version` 后保存； - -- 槽位唯一性：`test_hrl.py` 覆盖 channel 17–20 与 1–4 同时 clear/locate 不丢； -- 清除事件一次且仅一次：fallback 成功写 `_step_clears` 已有回归； -- 候选快照：`test_opportunity_measure_precedes_candidate_snapshot` 已覆盖； -- 奖励对齐：新增 `HRLReward.time_only()`，训练脚本默认 `--reward-profile time`； -- 终止/安全网：现有 `HierarchicalEnv` 回合测试继续通过； -- 训练/部署配置：`--config-profile production` 使用生产配置，`coarse-pretrain` - 显式映射到 fast 评分配置并写入模型元数据； -- 依赖守卫：全量测试 275 项在 dev+rl 环境通过。 - -## 6. 未完成 / P2 待办 - -- `scripts/pretrain_hrl_expert.py` 已建可 `--help` 的接口占位，专家轨迹 + BC 功能待实现； -- `scripts/ablate_hrl.py` 已建接口占位，无信息增益/无掩码/无课程/无粒子统计消融待实现； -- `scripts/tune_hrl.py` 已建接口占位，Optuna TPE 搜索待实现； -- Q3→Q4 迁移训练与配对实验； -- 完整 10–16 源课程训练、多个训练种子、冻结测试清单； -- 官方三次正式测试表：必须有真实官方案例码与导出日志后才能填写； - 本地 `referee` 结果只标 `local_self_test`。 diff --git a/docs/hrl-sol-execution.md b/docs/hrl-sol-execution.md deleted file mode 100644 index e2d6c74..0000000 --- a/docs/hrl-sol-execution.md +++ /dev/null @@ -1,267 +0,0 @@ -# HRL 续作执行说明：交给 Sol 的工程任务书 - -适用日期：2026-09-11。工作目录：本仓库根目录。先检查当前代码与本轮测试结果，再按 P0 → P1 → P2 → P3 顺序执行。 - -目标是交付可运行、可审计的整场调度学习系统；训练收益由完整场景验证决定，不把训练脚本存在或局部奖励上升视为完成。 -本说明更新了原粘贴 Prompt 与 `.zcode/plans/plan-sess_cf390556-4ef3-4ce6-b91e-782aee1718f9.md` 中不符合现有协议或证据的部分。 - -## 1. 现在有什么，缺什么 - -| 组件 | 当前状态 | Sol 的操作 | -| --- | --- | --- | -| `protocol/`、`referee/`、`harness.py` | 已有客户端、独立评测机、内存回环与计时审计 | 复用，不另建模拟器客户端 | -| `belief/`、`particles.py`、`planner/` | 已有保守候选、网格幸存集、覆盖证明、证书清除与后备清除 | 保持证明前提；改调度时复用 | -| `rl/`、`train_rl.py`、现有 RL 检查点 | 已有单源选点 PPO/MaskablePPO 与完整对照 | 作为局部策略和对照，勿命名为端到端 PPO | -| `hrl/constants.py`、`scheduler.py`、`planner/hrl_loop.py` | 已有高层候选、特征、执行与闭环骨架；存在本轮需修复的 P0 项 | 先完成第 4 节门槛 | -| `hrl/hierarchical_env.py`、`curriculum.py` | 已有训练环境和课程；奖励、终止、训练配置仍需审核 | 在长训前修正，覆盖独立环境/部署一致性 | -| `hrl/low_level_env.py`、`executor.py`、`train_hrl_low.py` | 有八方向到达实验；训练入口尚有掩码等缺陷 | 作为可选离线实验，不能卡住主线 | -| `tests/test_hrl.py` | 有接口和启发式 episode 测试 | 增补行为验证，不能只检验公式与实现一致 | -| `train_hrl_high.py`、`pretrain_hrl_expert.py` | 尚未实现 | P1 编写最小可训练闭环，P2 再加入 BC | -| `benchmark_hrl.py`、`ablate_hrl.py`、`tune_hrl.py` | 尚未实现 | 先评测，再训练，再做消融，最后受限调参 | -| CLI 的 `hrl` 策略和 `--hrl-model` | 当前入口尚未接入 | P1 接入并检验 `--help` 与离线闭环 | -| `output/hrl/` 的高层模型、曲线、评测表 | 本次审阅未发现这些产物 | 不声称已有 HRL 模型或训练成绩 | - -现有证据：修订单步 RL 的 Q3 独立验证均值比 `bayes` 低 0.69%，区间跨 0；Q4 比 `directional` 低 14.38%，比旧 PPO 低 2.26%，后一比较区间跨 0。 -这些来自固定模型、本地 18 场/题的验证；新 Q4 的后备清除次数更多，边界压力性能有退化。详情见 `docs/强化学习适用性审核与改进-20260911.md`。 -没有证据证明 HRL 已优于上述基线，没有证据支持“Q4 迁移节约 80% 训练成本”或“全部合法场景小于 4000 秒”。 - -## 2. 必须纠正的原 Prompt 假设 - -1. `/measure` 与 `/clear` 提交终点，移动按两次实际动作位置间的直线距离除以 5 计费；没有朝向、转弯或八方向移动协议。 -2. 八方向策略仅在本地推演再强制补上终点，无法影响模拟器实际路径；不能把这段推演称为比赛收益来源。 -3. `/clear` 不切换接收机频道，目标频道与当前接收机频道应分别记录；`near` 是测量结果，清除仍需独立请求。 -4. 接收半径逐源未知，动作掩码只能使用观测与保守界，不能读真实半径、源位置、类型或朝向。 -5. Q3 的 `no_signal` 只能按已证明的保守半径规则删格；Q4 的 `no_signal` 可能由方向遮挡造成，不能直接删空间候选。 -6. Q4 联合位置/半径/朝向粒子是后续研究选项；普通二维粒子加“角度门控”并不会自动得到正确联合后验。 -7. 固定地点误差固定；同地重测不是新独立样本。保留读数舍入裕量，不收紧真实的 ±1° 误差界来追求表面收敛。 -8. 源数最多 16 不等于只有 16 个频道；所有按频道编号索引的动作必须覆盖 1–20。 -9. 连续无信号次数不能作为全场结束条件；覆盖参数不能随意调到破坏已证明的发现覆盖。 -10. “无信息增益”“无动作掩码”“无粒子滤波”必须精确定义；不能通过取消清除证书或真源保留约束制造消融。 -11. 原 Prompt 的 6 个状态与实现里的状态枚举不一致；以 `ChannelStatus` 为准，额外 TRACKING/LOCATED 只能定义为派生特征。 -12. 协议文档 §6 仍写持续搜索默认开启；当前 `config.py` 为 `persistent_search=False`，CLI 用 `--persistent` 显式开启。先同步文档，不恢复过时默认值。 - -## 3. 推荐架构与工程边界 - -采用“高层学习调度 + 低层局部选点 + 协议执行”的分层架构。高层学习先处理哪个频道、继续覆盖还是定位、何时使用后备清除。 -低层默认调用成熟几何选点；可切换现有 `PointSelectionPolicy` 学习局部检测点，形成第二个学习层。直线移动与设备动作继续由协议客户端执行。 -Q3 首版低层用 `bayes` 几何选点；Q4 先保留 `directional` 的遮挡处理，待接线一致性通过后对比旧/新局部 PPO。 - -```text -SimulatorClient.state + belief + particles + coverage ledger - ↓ 同一份候选快照和动作掩码 -高层 MaskablePPO：选择频道/覆盖站/清除任务 - ↓ -局部选点：geometry | 已有 v2 PointSelectionPolicy - ↓ -证书/预算/覆盖约束 + 实际 measure/clear 宏动作 - ↓ -accepted 请求与响应 → 时间、信念、清除与覆盖账本 -``` - -八方向 PPO 保留为 `low_level_env.py` 的离线附录实验；生产默认无需加载。若论文要求字面双 PPO，用高层调度 PPO + 局部选点 PPO 对应真正可影响成绩的两个层次。 -现有高层一个动作可以包含整站逐频道扫描或多次后备清除，因此是变长宏动作。记录每个宏动作的实际持续时间、请求数和中断原因。 -先采用 `gamma=1` 与实际时间成本，避免把“每宏动作折扣一次”误当成“每秒折扣一次”；只有明确改变目标才引入按持续时间折扣。 -第一版保留 120 槽位 MLP，不同时重写注意力网络或连续动作；需要多候选/频道时另升动作版本并独立比较。 - -建议新增的共享接口如下；这些是待实现契约，避免训练和 CLI 分别拼接策略。 - -```python -@dataclass(frozen=True) -class HRLPolicyConfig: - local_selector: str = "geometry" # geometry | rl - local_model_path: str | None = None - -def make_hrl_loop(client, config, *, problem: int, - high_policy=None, policy_config=HRLPolicyConfig()) -> HRLLoop: ... - -def validate_episode(summary: dict, events: list[dict], - truth_channels: set[int]) -> dict: ... # 仅评测模块 - -def run_episode(scene: dict, variant: dict, *, output_dir: Path) -> dict: ... -``` - -## 4. P0：修复正确性，未过门槛不得长训 - -以下表中的状态以本轮提交和测试报告为准；Sol 接手时先查看 diff，已完成的项只做回归，不重复实现。 - -| 项目 | 精确入口与问题 | 目标与验收 | -| --- | --- | --- | -| P0.1 槽位冲突 | `constants.py`；`scheduler.build_candidates/heuristic_action`：clear 频道 17–20 与 locate 频道 1–4 重叠 | 本轮优先修复；每类按 20 频道分配，构造同时就绪冲突场景验证任务均保留 | -| P0.2 清除事件遗漏 | `scheduler._hrl_execute_fallback` 成功未写 `_step_clears` | 本轮优先修复；证书/后备/宏动作内部清除均由相同成功事件口径计数，一次且仅一次 | -| P0.3 候选过期 | `HRLLoop._coverage_phase` 在构建候选后 `_measure_at_stop` 可改变信念 | 本轮优先修复；选动作前候选/特征/掩码属于同一个状态快照 | -| P0.4 奖励对齐 | `HierarchicalEnv.step/_reward_from/HRLReward` | 改默认奖励，成功/异常/安全网/截断均计入已经发生的真实时间 | -| P0.5 回合终止 | `HierarchicalEnv.step/reset/close` | 实际完成、失败终止与采样截断分开；收尾不丢 /exit，不把零观测用于有效截断 bootstrap | -| P0.6 训练部署配置 | `curriculum.training_config` 默认网格 29.3m，部署 14.65m | 首个可比较模型使用生产几何配置；粗网格只能显式快速预训练，后接完整保真微调与评测 | -| P0.7 无进展循环 | `scheduler` 的 locate/probe/fallback 掩码及预算检查 | 去重、已清除/无效证书、后备预算不足在动作前处理；合法动作不能无限零成本自循环 | -| P0.8 测试与依赖 | `tests/test_hrl.py` 导入守卫、`policy.py/scheduler.py` 顶层 numpy | 不安装 RL 组时基础 CLI 可导入；完整 RL 环境运行测试时不得意外全部 skip | - -建议槽位布局为 clear 0–19、locate 20–39、station 40–88、fallback 89–108、保留 109–119；所有槽位互斥且小于 120。 -槽位含义改变必须升 `FEATURE_VERSION`，即使仍是 1248 维/120 动作，也必须拒绝旧语义检查点。 -模型元数据应要求 `level/problem/algorithm/feature_version/action_layout_version/reward_version`，并记录模型 SHA、配置、依赖版本、代码版本。 -缺字段不能默认为兼容；Q3 权重初始化 Q4 必须走显式训练迁移参数，部署加载仍严格检查题号。 - -默认奖励先用：`r = -实际本步虚拟时间增量 / 100`；证明完成时加固定完成奖，未完成的真实终止加失败罚。 -失败惩罚按冻结的 episode 最大成本界设置并记录，不随意用不足以覆盖数千秒的 `-100`。 -相同全清场景里逐源常数奖可能只是常数偏移，但不同终止路径、事件漏记和失败场景会改变激励；故首版去掉新发现/熵减/每源清除附加奖。 -如需塑形，另做势函数差 `gamma * Phi(next_state) - Phi(state)`，真正终止态统一势函数值，并检验总和望远镜消去。 -如果优化题目平均时间，用整场时间作为训练目标时明确这是所选代理；不同源数的跨场景均值与 `T/N` 均值不等价，不能暗称完全一致。 -人工步数上限若仅为采样截断，返回可用的终末观测并允许 bootstrap；若定义为任务失败期限，应作为真实终止计罚。按该语义设计测试。 - -P0 验收至少覆盖以下行为，不只验证 reward 按代码相加： - -- 20 个频道的各类槽位唯一；频道 17–20 与 1–4 同时有 clear/locate 不丢任务。 -- 同一观测轨迹分别构建独立训练环境与部署闭环，比较候选、mask、特征、下一实际请求及状态。 -- 固定动作序列跨成功、失败、后备与安全网路径：reward 的时间分项总和等于 accepted 日志复算时间。 -- Q4 保证盘内/盘外无信号不误删真源；正读数重置遮挡序列；无效证书不能执行清除。 -- 10/16 源、频道含 20、边界源与预算中断均检查真实源集合、缺席集合、正常退出、异常与计时。 -- 评测端真值仅进 `validate_episode`，策略构建、候选、mask、观测与模型选择均不得读取真值。 - -## 5. P1：最小完整闭环，先评测再短训 - -先实现 `scripts/benchmark_hrl.py` 与共享审计函数，使 `HRLLoop(high_policy=None)` 可以与原 `bayes/directional` 同场景对比。 -这个启发式 HRL 骨架是重要对照：若差距来自任务候选、机会测量、Q4 保底或预算语义变化，先修或如实量化，不归因给神经网络。 -每次运行将 scene、误差种子、配置、summary、完整 JSONL/accepted 事件、时间复算和模型 SHA 写入新目录；目录已存在则拒绝覆盖。 -单场异常也写结果；聚合存在任一无效局时 `all_valid=false` 且脚本非零退出，不从平均值表中静默删除失败场景。 - -随后实现 `scripts/train_hrl_high.py`，第一版提供以下参数并通过 `--help` 验证： - -```text ---problem {3,4} --stage {1,2,3,4} --timesteps INT --seed INT ---envs INT --vec-env {dummy,subproc} --device {cpu,cuda,mps,auto} ---config-profile {production,coarse-pretrain} ---model PATH --resume PATH --eval-manifest PATH --verbose INT ---local-selector {geometry,rl} --local-model PATH -``` - -阶段 1 即保持 20 频道与固定观测维度，只改变布景复杂度；早期课程可减少源数，但完整训练必须覆盖 10/13/16 源与实际 ±1° 固定地点误差。 -Q4 完整课程同时覆盖少量/半数/大量定向源及近边界向外朝向；Q3→Q4 迁移与 Q4 随机初始化作为配对实验。 -禁止让策略知道课程场景真实源数或隐藏的占用频道列表；初始位置保持官方 `(0,0)`，除非任务被明确标为局部交接课程。 -先测 1/2/4 个环境的采样吞吐、内存和每次决策耗时，再选择 Dummy/Subproc；小 MLP 不假定 GPU 更快。 -Subproc 环境类内部提供 `action_masks()`；Windows/macOS 使用入口守卫及可序列化工厂。 -随机种子覆盖 Python/NumPy/Torch、PPO、每个环境、场景误差；每个 worker 使用独立 Monitor 文件。 -保存实际 `model.num_timesteps` 与请求步数，恢复时说明增量还是累计；续训加载正确算法/特征版本且不覆盖原模型。 -使用掩码评估接口，如 `MaskableEvalCallback`；同时在确定性推理中传入同源 mask。 - -CLI 加入 `--strategy hrl`、`--q4-strategy hrl`、`--hrl-model` 与局部策略选项，经 `make_hrl_loop` 统一构建；默认策略保持现状。 -显式指定缺失模型应报可操作错误，不能把启发式降级结果记成 HRL 模型成绩；启发式骨架单独命名 `hrl_heuristic`。 - -P1 门槛：启发式骨架全场审计通过；512–2048 步训练冒烟可保存、重载、确定性推理；CLI 内存回环至少 Q3/Q4 各一场通过。 -短训先用单个种子验证流程，不用这次模型的成功率作论文结论；完成后记录吞吐再定时间预算，不预设 1–2 小时足够。 - -## 6. P2：完整保真训练、独立评测与消融 - -实现 `pretrain_hrl_expert.py`：运行 `hrl_heuristic`，保存 `(episode_id, state, candidates, mask, action)`；按 episode 划分训练/验证。 -BC 与 PPO 的网络层、激活、归一化、mask 处理保持一致；复制后在真实样本上逐位比较 logits，critic 单独初始化。 -记录专家动作无法映射到候选的比率与原因；若大量回退到最近候选，先改专家映射，不用假专家标签继续训练。 -至少比较 BC+PPO 与随机初始化 PPO；BC 准确率单独报告，不能代替独立闭环用时与有效全清率。 - -冻结数据集分工： - -| 集合 | 用途与约束 | -| --- | --- | -| 已有 700–702 开发场景 | 查实现错误、探索配置与性能分解 | -| 已有 91000–91005 场景 | 历史回归；结果已经被审阅，不再宣称是本次全新盲测 | -| 新开发/选择清单 | 固定 JSON manifest，记录 seed、源数、定向组成、error_seed；用于训练选择和调参 | -| 新冻结最终测试清单 | 与训练、BC、调参及选择清单不重叠；模型冻结后运行，失败后修改则新开一轮测试 | -| 压力场景 | 圆周边界、1000m 半径、向外/切向定向、高频道、固定误差与预算边界；单独汇总 | - -同场景每个变体用相同位置、频道、半径、朝向和地点误差函数。每个模型至少跑 30 个完整场景，再按预算扩大；“30”是工程起点，不是统计充分性证明。 -最终用于稳定收益结论的主方法至少 5 个独立训练种子；场景种子与训练种子分开报告，不能用 5 个场景替代 5 次训练。 -Q3、Q4 分开统计，并按源数/定向占比分层；所有基线与候选使用相同停止、预算和计时口径。 - -| 指标 | 固定定义 | -| --- | --- | -| 有效全清 | 成功清除频道集合等于评测真值；缺席集合与真值不交；completed、正常 exit、无异常、计时一致 | -| `virtual_time_s` | 实际退出前的完整虚拟时间；主竞赛口径显式 `persistent_search=False` | -| `avg_locate_clear_time_s` | 每场 `T / 实际清除数`；有效全清主表再做逐场平均，另报源数分组 | -| `accounted_virtual_time_s` | 首次完成证明的时间；与最终时间并列，不悄悄用它替换正式计分 | -| 运行时间 | 每场实际墙钟、模型推理耗时分位数；本地内存回环时间不冒充官方程序运行时间 | -| 成本分解 | 移动秒、测量秒、换频秒、成功/失败 clear 秒、后备尝试、重复测量、总请求数 | -| 尾部与失败 | 中位数/P90/P95/最大值、有效全清率、失败明细；小样本尾分位注明估计局限 | - -平均 `T/N` 不等于 `平均 T / 平均 N`；若另用总体总时间/总源数，明确标为按源数加权口径。 -配对效应以同场景时间差为基础，报告均值相对变化、胜出场数、95% 配对 bootstrap 区间；相同场景家族共享种子时按家族重采样。 -多训练种子结果报告种子间分散程度；必要时对训练种子和场景做分层重采样，不能用单一场景区间代表训练稳定性。 - -最小比较矩阵：`bayes/directional`、旧局部 PPO、修订局部 MaskablePPO、`hrl_heuristic`、高层 PPO+几何、后续高层 PPO+局部 PPO。 -此矩阵先回答“改调度是否有用”和“增加局部学习是否有用”；既有单步 RL 不能命名为无几何的端到端基线。 -若仍要端到端 PPO，必须单独定义输入、实际动作、安全执行边界并训练，标记预算；它不是 P1 完成条件。 - -每项消融使用相同训练步数/场景预算/种子和评测集，单次只改变一个因素： - -- 调度：学习高层 vs `hrl_heuristic`，保持候选生成和局部选点一致。 -- 局部策略：同一高层与执行接口下，geometry vs 现有局部 PPO；报告是冻结替换还是联合重训。 -- 塑形：纯时间目标 vs 经过恒等式核验的势函数塑形；无信息增益奖励就是系数为 0。 -- 课程：相同总交互步数的课程训练 vs 从完整场景开始训练；Q4 迁移另列，不预设节约百分比。 -- 掩码：只取消重复/低效候选等策略性屏蔽；协议合法性、证书和预算硬守卫始终保留，标明实际取消范围。 -- 粒子特征：保留几何守恒内核，先将粒子统计输入和评分消融；若真的移除粒子模块，另实现保守几何替代并重新验证。 -- 八方向：只报告离线到达率/路径效率/推演耗时，以及实际协议终点相同时成绩相同的事实。 - -P2 推广门槛：所有最终与压力场景有效，Q4 相对最强已冻结基线的时间差区间上界小于 0；尾部容忍度先写入 manifest。 -建议初始尾部门槛为 P95 不退化超过 5%，这是可调整的工程标准，不是题设要求；任何调整都在重跑最终测试前完成。 -Q3 若未显示稳定收益，保留 `bayes` 默认并交付负结果。仅比手写基线快、却未优于现有 PPO，不能宣布替换默认模型。 - -## 7. P3：受限调参、论文与官方验证 - -只有 P2 主方法具备竞争力才实现并运行 `tune_hrl.py`。先搜索学习率、熵系数、网络宽度、GAE、合法候选排序参数。 -不搜索真实误差上限、光学清除半径、无信号停止次数或破坏覆盖证明的网格间距；确定性网格没有粒子重采样阈值可调。 -Optuna 只用开发/选择清单，失败配置先按有效性淘汰；预算与失败/剪枝逻辑写入结果。最终模型重新用固定配置多种子训练。 -文档、流程图与论文表明确：学习负责哪些决策、证明由哪些模块提供、本地生成分布未知是否对应官方分布。 -训练曲线、实际步数、配置、全部对照与失败明细均归档；不要只挑最好种子或去掉不利的边界场景。 -官方三次测试需要对应演练已经启动、真实案例码与官方导出日志；本地 referee 的三场不能填写成“三次正式测试”。 -不覆盖旧官方归档；官方结果与本地结果分表，不对不同官方案例直接计算改进百分比。 - -## 8. 命令：已经存在与待实现的严格区分 - -以下入口当前已经存在；依赖需先安装，命令失败要修复或如实报告，不能跳过错误声称通过： - -```bash -uv sync --group dev --group rl -uv run python -m pytest -q --tb=short -uv run python -m pytest -q tests/test_hrl.py --tb=short -uv run mcm26b run --help -uv run mcm26b run --robot-id 2026001 --self-test --strategy bayes -uv run mcm26b run --robot-id 2026001 --self-test --problem 4 --q4-strategy directional -uv run python scripts/train_rl.py --help -uv run python scripts/train_hrl_low.py --help -``` - -`train_hrl_low.py` 目前仅 `--help` 可用于检查入口，不把完整训练列为已可用： -其 `MaskablePPO.learn()` 需要环境掩码，当前 `LowLevelEnv` 没有 `action_masks()`；改普通 PPO 或提供全 True 掩码后再试。 -还需修 BC 按 episode 划分、统一随机种子、每 worker 独立 Monitor 文件、评估 seed 生效、实际路径长度与成功率分开统计。 -低层到达失败不能靠 executor 最后强制补终点伪装成策略到达；环境裁剪后的实际移动长度也不能简单记为 `steps*100m`。 - -下面是 P1/P2 实现后必须支持并实测的命令契约，当前尚不能直接运行： - -```bash -# P1：先验证不加载模型的共享调度闭环 -uv run python scripts/benchmark_hrl.py --variants hrl_heuristic,bayes,directional --manifest output/hrl/manifests/dev.json --output output/hrl/p1-heuristic -# P1：完整保真冒烟；模型目录必须是新目录 -uv run python scripts/train_hrl_high.py --problem 3 --stage 1 --timesteps 2048 --seed 320001 --envs 1 --vec-env dummy --device cpu --config-profile production --model output/hrl/p1-smoke/q3.zip --verbose 0 -# P1：CLI 明确加载高层模型 -uv run mcm26b run --robot-id 2026001 --self-test --strategy hrl --hrl-model output/hrl/p1-smoke/q3.zip -# P2：正式阶段训练；步数先依据采样吞吐与学习曲线再调整 -uv run python scripts/train_hrl_high.py --problem 4 --stage 4 --timesteps 60000 --seed 420001 --envs 2 --vec-env dummy --device cpu --config-profile production --model output/hrl/p2/q4-seed420001.zip --eval-manifest output/hrl/manifests/select.json --verbose 0 -# P2：冻结模型完整对照；variants 的模型位置来自同目录的配置清单 -uv run python scripts/benchmark_hrl.py --variants hrl,hrl_heuristic,legacy,revised,baseline --manifest output/hrl/manifests/final.json --output output/hrl/p2-final -``` - -命令使用的 manifests 也要由脚本生成、校验并归档；尚不存在的模型不能当作初始化输入。 -添加 `--full` 一键课程入口时，它应展开并记录每阶段命令/配置/检查点链，失败即停，可继续，不是忽略错误的 shell 包装。 -参考现有依赖配置：基础运行依赖仅 requests；RL 为可选组。固定环境版本并记录硬件，勿假定当前执行机就是旧报告中的 RTX 4060。 - -## 9. 可以直接粘贴给 Sol 的续作 Prompt - -```text -继续当前仓库的 HRL 工作。先读 docs/hrl-sol-execution.md、最新测试结果与 git diff，保留现有修改。 -已有客户端/referee/几何/PPO 都要复用；不要重写骨架，不把缺失脚本或 CLI 说成已完成。 -先完成 P0 尚未关闭的奖励、终止、完整保真配置、掩码和依赖问题；本轮已修的槽位/清除事件/候选过期只回归。 -采用高层 MaskablePPO 调度 + 几何/已有局部 PPO 选点 + 协议直线执行；八方向只作为可选离线实验。 -按 P1 先实现 benchmark_hrl 与真值隔离审计，再实现 train_hrl_high、模型元数据和 CLI，跑可重载的短训冒烟。 -P0/P1 没通过就修实现，不用加训练步数掩盖问题。通过后再开展 BC、完整保真课程、多训练种子、冻结场景配对评测。 -保持 Q3 bayes/Q4 现有默认策略；完整实验胜出前不覆盖默认模型。主指标包含有效全清、T、逐场 T/N、请求分解和尾部。 -每阶段交付具体文件、实际执行命令、测试结果、模型/日志路径与未完成项；不预设 Q4 80% 节约或 4000 秒保证。 -官方测试只有真实演练开始并获得案例码/导出日志才记正式结果，本地实验单独标注。 -``` - -接口参考：[MaskablePPO 官方文档](https://sb3-contrib.readthedocs.io/en/master/modules/ppo_mask.html)，[Gymnasium 终止与时间限制](https://gymnasium.farama.org/tutorials/gymnasium_basics/handling_time_limits/)。 diff --git a/docs/paper/main.pdf b/docs/paper/main.pdf index 81bd093bfdb9f5b6e7b05692eb4724b1d3f68dcc..5604decd67eeb151a222aae74befc4d399e04598 100644 GIT binary patch delta 37718 zcmY(qQ*&j&1Ah*tYFAwr$(CZJlqAeJ=ieyDsNiqsCLSYK7zD;b9Cora-AX$mP~7%^+Uk>xP7~+W8u!#{cGgN6VTIrO&D7`VCw7n?X_0Z zF>{f6a3+(vwbu&pCG~qzL^`Rwc>fr9`x;ni@A$sm0DSap|9m}s-JPvGy-t+IiiU3e z^-Urt>d9Wo_B`lt`?)%D^=;?n7r3#rt9I+S5af(ztVy-+aG%(Fdw5^?ycsyE)sHD< z>~POL=l|IR@aO*MjpCl~Z;l%T4l7iOB{(xmXv5axI#9vg1DvLHlBaE7s;Zo%< zG(qUpo+Jm;V0KO`wZxUrLW{^#iTFvJZTsB)7Sa%g$sP!wRwKU2w+Xr(((ELJsR>N+ zt%4CRvmTua>zo%uHwwh>26yoy=P}ND?sU^H1Hh3}!9nDci^f|R$NPr+N<)>QDc7^2 zZBwzt<1a@Kd)>pCjGB|U*^@ml)|z8pnnprY*2WbE(BVu^tJxG% zpyx}ZU>@WP1l=yk3}HqxD(p>&yx1%T&+a!`z-YNcUQ8hyU=Oa4p((S>mF32&^i>ar60TOJsWkIJKy{G`=L0$pAiS#0@57Q1fwu{8oxoWL`SCrabjw$=mE23O4;e|^bL&f3`xY(SI_rGrV+qP3_;G81bJKx?P=9x45XIMA|l9T zAGK-9+ZU|c?~9{)etA#Ihtuqvg#*j4?<4`i+!KcG9}t6=(D~H8Mi8{r@Nf`n0Q>(d z0lpc3D)l9uLT;XDpO4`CEL>}rD47BT0vHpdN$^CjP0ku5n6^ zxdJ|3KTC1nKi?++cLV*d>!0`QiJq>Hx4Xyp(4X(vpZATIpZn*JmfT!HzpuxsjqfRd zfWY^AtUKe*=k1r%-PynakA-iFS` z({pAdaq)+wvOzC2$SbSE7xpZS^MOqEHsgUUvo5L0^PHgzXHQ+2#1{G{yeI|B&N$OH z-KsxKumxOhGuHeye#$vhEYiPpuie1~CJ0>+{!uVqalFxwVHbMwqb5LbIqTRlhnzNK zy=gG#w57A8qnoAIhQIp79-YnckV5xUw#_m>Yv+n<~w_M{oJYA0uUs`2p6wWRZH9IIU+M7d-O7Pnj0FGqwqt`UVS)I5OEtsg!H=1YY4c0b3Whb>IS_ z|AN_DT1^(m#K1_n$y*9fa!ixiRjT4O#|{T}<_Tf_v#c8cC-SnngwIYNsGY^PE5s;X zpG2R49JSA_j8Zu0?>1rkyEx+tQ`>~420$ZPwwiHY!5$B!8(ZV{?z)wK)OOVYICgv8 z#5s#k)A+WJ`njGH+9bQ}E=<}ejCNb!|MU4Rn@Haj`IX@o+tOuY#sT^L6XnCc=E#uM zd4`$Mf-waA?F=X1d@yO9d0muyTC=4?-FI0ic{ZwXrp@8!WYA?4d%4mS!3lR+4;Vpj z%NMBg(7*|GNt)Y|589qFWN@pPceZz&2U^j83J{RElH+0K5HXLXi<`B7-wsdSx}l`bXYice=#Bgp zkhs%08jj9wgoh<{SGsonokU>#)8~wozTkCPTnK~ohjN)u(Zl}VZerv@#KVi+As?&U zG@?1w+G@aq>%!iCqZ^XZ+or^?^jusWF$o1ck~_pHC~c=SDT0t|5Ip$!RscJKW2RUI zGG0QR89ooMBR836?_)^g@#ttmQR+2UhZZNHgTyEnD|hal>E9Q+Y8WOdWbY8|wS0@7 zg-2~V(K_Z=C&z9Qv(pmtS0};{NcTa%{RyVsYo)QUQ7@4t+v_5@q76QhouwcIrv-I$ z`@s@o%uUBgJT$oBLoRQ)S^(rnpCPqV!SbD&!Xkmqz_nhfE0(p}^)g?nvG%PEa))^2 zweV%LEfVd`{1Y4ArG-1~y&?{G*hVuW39F48>c&7^i-FsW!v=G_r{Muw`jCsay=ZOZ z+Uw(?sD-NbYco3=#WpR}nrIr_&oSTO3_b3>7GFKxWnsg3Le_&=Js_DaQ3~6z&uQJ= zC@v#Tj;?iO(el6_BIK7$tv0x&*+4=8cBNJbR`&D?BNFVmq7f){{~K)}dZt$=HW z)$AE>DOqQTyO8{h5}ZO8B&%1McV`KO zP*WEF8qCj`h?ves^tjS%IpPjRGaY?Z__GjHN5S%xEt@VOtfivg+119T;hn=omw9q# ztQX+a8CY5ZdmjzN?CElXT1Wi?DbXk}S7M9xSl=mXfSTcdJ4He+!LEtwWfIDJaevK;pTb=Vz&Xg1Z;DZFLe5=o;)WL zn9A|wv?|;MGN_D6aEj97|6~+KwLx3LtM5)7f8(sll!(Z)qtVWak$U&Q&uO#3+1%2# z*;%Ep0<4XVaxTOa>XTX5vD<>S%ruW(kd0B_#w;}qmoyEb?i1~-EBe=g6EuPEX+Yv| zm}W`RCnjstseE-rA7=Uc-583)^gkVwYKJ@TLW~igc)ChK zb&}lgS|)?B`>8AUNwp+~RXk`Q@aMWN2B_CttT56`@7>k_)*{oKtZW z0DJn#kqD-Oeli?z6@h)9`=cCk^lN(dCo0~#*cj&Gn1+Df$d)1?2SfU{<@qD+S^v-_ zoiN1SA_F#W<7!X!PR#heoPf2B-wV5P)%{WBZc?s$8?VD%tZJK>G|au@c!1-Dg!(pdj2CNtrK4&LRAmRrVQi zaJdyn6Cky0ptfWmxpFO@{9%>_3Hu0{c6OG8!VSwzzt<$*^!^{IGkALcmZjI+^_eW1 z!5#B~+gh0XN*3Zzmk*3{9wiR?2 zCM&)dn}N^?S7z@)$rZMa3rqQcw=wJ*m;tCb8}|fK%`SSIN@J)y=jk~!6|@Dca)(-3 z9I|qL=Bn}feY#7+urjrF^(Gm@L!f$RDvh>BF`;y7w6-QfdBef>8pw9#VT1Z|BLA9- zIwy}dg8tvWzg{SC*LaCn8c*W&r2rj_w?Jkmb;aM1?N-Uq0$~!av*pKt;qxMAU3kxP z8x53DvK^uQxTMjx^8#M83}Qg{A4tIXs5~=|R?ezN>bObL)Lv<)HoGg#g0yDa-F9MJ zk*AYld%8Tk1MH?VuP4!C2#X%7#$NGoymB*Kw0d^l*_Wv^0&Ae_3XZBGy$r<2@ zC(D3XX#)mL;jVDjD8ekDXdg;_$uwTK9Ayp{bMA1iL&P*$W?D35VbOV=ZgfqO7)y-B z)k~x>!qN+2%Q14{e2CTS9nV%~y=hjX^%+igZXFEwaVfTl8RwP91#RezaOP&yDag%@>_E+VBQHH|40m2&6p4H;tpSCP~S&Z;=H;P!o!IcV!+DBwGDVAeA$^QTy&BrLq3j z-;hZu*I{+_@Kqt_&}S^MaiGM#6guRr%#4-^aNnbb3l9fy)dP=vQ!s1D6X|^J>IqZb z48mIse7sCA`~rcJq6jTOOj0~uuhQ{g13{oTLwn@u&(P@F7whki0YAIExHgobHgLxg z%R^LQK3D0#c#d^m*sdc(Y2#MzpWmZ>S_Qq1Xk_9<{$Gdv)C_Sd#aE4&D;_|uwVDP| z^Qb|i@-x5zqMwIjS*Mc%`JZVT{4CYoRt-}*g~o|kVll2!*uQwLCh=K_<480jocCoIy=`68XM?TYII3!niCrN(d_+F&h{F2M~&p}07UwsV- zl^&?2C5^+L5W`MoZKt2ozv2S90e8Ku+DH3;o0$O=GMfT}8A~|~MZ)$|0^Dg2Z2P4V z@b34C;nQo1>=Y#7865@uZ5LvxI$&d!OZy&q8&d9gsPer01nqj*w+=No{dwpMiG_uG zS)Ed|;?`LCSUN4m6|YWpXkHjf1oY!394M56148C6ys;Tb?+omOzfr@8ez+j;Q<1Xs zE1Ljk^&&%vh&@Q2N{DRhh%9d{UcY_TkgDK7CYq8hs3@6d-)s| z3I<+^A9_V`yr8#s3xi0s@LTVX$A?eXpL%`H=J2CeKp3_-Sr+fLXS8#2k|!nyNFwGmro2Ei7>&$h03bu$$6`y}W0%<+7)Bv1U67`jP1 zBUWKYefjAoCV5*wAyt~hS7P{CJ)r)d%@boziPXl`#7;lL#KDN4Cwj5tG4)H4FbjzE zEIch50QsJ60X4}>hD^#IUMi3x2Q|h~9&A4&kL{l9gt+?cm&&L&xaDA4UY;}Z${|e} zqu#&cFlPE$9CWp)RaEdN@NTNt0urV3F(zTXf|7vtJ-*hW-O^#75Ew65KdVz53M#T& zapPxjKdVWGxzdpKet}1`HyXK9^1u9C`L#&6!V1_FT=_~14(3Xnd{^M$u3$6pZaPy! zr$dvU!iQ*__+xuCTXrg*j@5#z-`#JttXbFV^H(y>^1LQ*FGz!fr<(tJmpvIEB7gzI zB-tZK38gk>p2_}E1n1=9ddb<8AUdMYmqZ-p1qAxq-$`%(d8%#I>IDw_nbMEi?+Z8X z%j1t*CmtgEhmWHJx3ticwgBVSucz3r4L)FtVo&mxV9#}q zTk6Bb*JG_L(%t>l>+Msy3@`o(7C_(cy>mKH=a%C}VCUvxwzUSk?Ez7?2;o|nldVMc*Yvk$O)-QABMHt9|*P1wa?Z7>Kv-0(~`)a1< z;as_K#>@kF)u26fa-evk0=>@Pga5s?v+k>B(s2VM;uLDnuVbSWB2}h<1t4fsf-1qy zPFG~{;EMNH_Ww}zD=?PL%7Q0UdowSAmR~QBDf_c^QuVQ3VJjj-cuXQR+3%Gy;zDDWT(GR<)}z%xL%XYPrFG}n~>DRPaKBJy(F-6dpIzXDoc$z|HXr7 zd!Qp}*V`Z3RARztB1{a<2FQqQ$vV&%$JC3tBFxN^qJfcM|JGJ~WkdbK(`F?lU8R9C z@d7t+CKCZywJ>MxtVJ4r$>3(CD43o}3OE0Ib`{_SVLKiIy%|ZI;WZV2lr!o-0R5C! zrlltj&NwYY;=!DOl3)^h3XxpvWfX)TpI(SX#R5$;kN!AQ-(JGw05G?&n%pIJSZ?&+ zX30^XHJ?g{-ekwo4m=$emR%O9%1Ew7Y&%8Kt2@LBBt7T!jpCDSEeBPzYY;i2JXV3H zsUeRyCr@*2D$@?i9#GS!$49wOBCR+q3J`&xAuBOnn9Au`XKi-1qTIP9)i$@97ljes{-lduZwd9TgbBy&BaSu(<- z&=66~WHyg9Vw8)|YJ3wUeHBAcqM1mrS8$he(~ zsw(%iQPD!LnocYCx;ak(!NXx{Ern-Gn;KQWfJ-z+tc-YCfR9{Er&-8DiN+#&@M>H# zRM3PdGGq$6VrhSmUC_Ijd{(SBvxMReZbc_FmFFgrMYPa-kHO4ZvuKyNUhwdb-eY-? z%98r2xj2u2xPlQ_H(gPb$lejq+^&}^GecRU5Um2qprW~BM_~!Wpa$}Dk^2l4dBnK} z&=+__x1=~pK#mgD2t+y@_$f;P*34ar7Ocjt7JDmJSi>5KI&iJI%gcqD&5tGt*S{i^ z%M>dw2`yFS9HmI4N`6T36q{I*K8P$M3?2D0DXuddVoizUUQDmzAm~Z=?%`dO?hC3* zP^`O06p^Uqk2mISQvj4C055C$;RR~y$>~bt^#f@Y%pAh&f7Ady6*(7#62S8RE<*K` z$E*gJkh?x;kJD5F=vTB6hi7>Zurq>JzkiA2mIhb|Yf!y?tS4NELXq8>d9psBr!G~i z^b7kmXDprOp+)?;d>UX0qTLRPkb>1^;`|L@m6k_Og|doKBN;3`-X!eN$NOC;xIZH< zTQU7~FS_%V572I&$^rTtW65Nc3QJMyUY(oYC}~tI#GRlB`QY$8P7D0Kn|{JDkk}b> zY82TroJ(nXSwu3vrc&mXDk!*hA)6d!8`)M0NvXh-@MO^fdGo6!5gGa!YEW{$TIFmq zc1uB6ldh)-mfZNKL0BavlnYYDw_cRvZbS_5arQ=qrW_j~V^p(as%%RMHoUw_nl{QyvvWTB#m(%b zXeZNIeQp07Nv0w* zN)}`OpsNO|sV#;OVnjsX1~(R|ktbsTWRFymbx0DEWDjZDio}2k8l%Z9b@h$9y5(x> zQdY$>80iTSrYhWX#YyOp=#_q6SxcJ48t53K==&s=`C2Jioy^qS+@^oT zi@n@f{9N}kUJ)!U%fs)e$Hpx0aE?$R;uA-1P%CA~j|N$^(R1~bsglbU?dC`D5(PtdOf za0o86w#BwCc($e2r7C67je%}}d4}^}u5sg~TQ0m!B&S&pRtU-Bdig8!IkjO-81~dP z3_ju>#;-R6_7Z&kNGClsWvHLZOhaTWXz|?f>!Xp%7hI}xc}PodXR+Laa6l2){%wI< zuO%hQ41uv4RZ)3)qQ?lMS?0zc$@k${zDBIIAklH*Hr?{138-XI(ZE1}D4p{Dvtvgm zo6gYEZ7UZW-5``wPRW4HlCZ9-YJSLIrvIFy4Q9YvsqHAkR;!p8RIbbGJW)v1st z6(c6bzDVVerGt=e=)@I@KgPP8TO@|P7~XSFj9Wo=otvb4>DHW;tu+|s!CV_V5HU)k zD&;qEZ*0u#oHpZ>_^T&ilgC9nEac7IEhuyjxd+OJojtGG{0^EBWpnK}ZlC1s=w?hC z<4Mf{+SV(UU((@8R1mFTenbbiBnIK`LDef^Cw}R!eAtl4qbv$;T50bDB`Y6#xY1l! zn4;f>_^KzbESGt;y~4ZsAA##2&cjr-Yv=rt>v~a1ivdUr4%{ig!ij^XP4;vV6agFW zqaOVx_a@Fpc}oBSb}a8`&cOSg^nj+{N;E7iNZ0;2=vIDUdV6VKzXeb4&@(7ZSN8y5r~3^if@@>WtRVm`i=M;z z;-Sl9j~iy)=)89I{NJ3V5GiQG$5~xz=aFb}O$u|%#oKvY&qsh;BdP8tG(UGQJs|Us zK<~k&g;MJcV}{+@#F}~ihtt2l_%kkeNuxe?R31d0W(2XI@qs&(8iK$>R|d)(sZ@| zMu;|U*E&gvm~1BU0bM|0F}NN#fP7%z-s=%$fGFnU9EH z&hW@z>a!#w6A=8fit`;QMK3{lgjQYpr#F9%TOW#8UIPIc-Z^TJs)oh4tz#O)3XEx~ z_db_^k;iP;8?+LTyz917N#l(*{-1~eM!VqR9>}P zMg`8QWC+F<{64tV+TGONP@ znU3GnM}vJdK?nF`@ZG$*crA0?<+n;SvnmV#wqVZ4cw^Tg6DjS_3LrZSixQn8k}!0? zPE=1qh03*8?PzqdL{&RX1g7kkw^15A*Tlf9_NQ>CVAOqA_Sk5<5(kKWTW{8hTtlAo z4W`fMKTp|O0)s2=Gj2Ink|Y`2lm{UmIGqQCLMUtr%eMnJ-QI2GiIRA98UF;!Y2P$vjT47`zl>3WE zkZ&;e4G3LM`$joL^i&JGg~*%PKpOsz81zWWDuH_;+Dgjq7z7`P)wchVBrCc0IEJC# zh6leh5Bg!GIPK-ny>Dd)kqCRv;f({32W)wA;g&WXW)v6and+?PB@--oTTdU9HM|gv z0TTb|hk;wn{Zoaah=+{8tvq_FpwVVH{C8RLXuou_*rVDNK<;A=yw#m6!|0b_V8)gb zZbpd7CgdvRN2ASTAHF^#3g!<8%=hWt5KUf(+9GLmrAP;uE5nJr98rYN zZbWPhaR>#PHXTJzPcoRirR_VU@>AdW$5^oi5SQ^qN!IR-#61OkbC%3i9Zy(VioK`#d39;4%yZD#2?I}e=eP3*BkDN-Ihj}{k z5P)(W)%>gaX9(?H6cXInQk2ot{DdHQ7&+(BF)k94O%jZ^l5beA-UWqYO~_kX&-Ugt z&-MRHlaQ7F@%R7jBE?VG1u`LvzkEk%Q&iB@7K()j#X(b}S}jNt6a9J?ijSy+r+fW) z7j8HOJ$`-nGUG#>|4f*$3)odf>gQy`PT+DoE3*$r0+m#ETuio!7yFAXnqO#OXD@D<*^ zx0B4FV3idW9J=aYl+H47X~6|A&*XFOQ}6Ta!_nEn#o;gJU2{{_{n8H8{H?5o zgEMzuD+BG111bpTh?9W$T%`GJ3c%}%$yBmbaqY!3kZ`gnh>gB6-Sf~QzJ*SPpi_?l z4k<-OvsJ3tPJjGVOx?sINT;4`P`Z9|%lC;TJX7tQVT~-coJ2GL19ivHHAs#Sc#Nc+ z@XJ)HNG_X7dmklVL-J>BiT(HQK5{L>ivqRa8h0HE=`b}(sZY~77WGa#wf~7Q@>5U58M0G_-$HdsP)#{TbW|DjMa`;;XVESYUxWMnkzSGJ0mOZ9vWgQ1 zKsAjW*cMp($dd5pa1!w~K-Tp3CK%_WhD7@oKErP3q`>4Hi?1I|(LaF?3mP21reWop z1#@AJ@$o6<`eRoSV>`Qi?E9x+FR%=k!(r@ArnSH9o(dNB+X$MaDz32(^Ci8wX#%cj@A4uA7)8@DgIT^Rwq1>t{BqDtCGFQX!{glfgV0q9x{Zre{i?JmZA z$w5A6c!e~J=x2a)8I4a=3LX(sRkfA197q*|4i~cyBBlF&E`spHm-iQ2b#(@us92aB zRGc)NRikmPNVlaJT#kQ~yBj6KJqvp$y%7p+Iih!&Z!f zsv={B39*&(IT$nf0r1!Q{tDih<>W&6y;V`GF4EPg8+}#e|L)=LVTXRCBiVhnS-E2U z{ayI@e-*~oRe9Uv=GjyU$inl?YEpD*EsAdX$93pRKoMkNdD!*af5Y39e(j|c)o=2z1?$B4 zYn>!;)-6Qk0QjD4So?@X;56zm#O7(2(^Puc>O66Phz~qObcSx$uw=@9Mu6{?#}4Af z(4?(#_xmPq?8k6@8dm|_{+qf*v5|TZ#WmV14F6aDZn!_4G5x?sWe@F;nf~{jnA1Nx zdK<49IZi|kir`iqECYxdd=UdQEJW`g?o{UTeaEtB1n5~rpjOQi*kdXdw*^z?7}28n zzEOX>NcFxMBI#$h7foiLjY+b83L}QS!s1xYo@>oJAYO1*;#Qa5^zjq{)M zDJ>}ZRY>wC&riuDoF( z{wchaaD@vO!YSb=5rr*^pL8CfHc(vuQamoc0YJPaYfq|)3e#kc(Xu^2pXI>2tL?&; zQ_1CCk5Nzsxq80jm!A6YV99Oyx}arG{vGcbrFKW>7+Sp%sPZpU5){fNpaD1_uc{52i~G+{nN z29S#^J2wAva*mAUY|;bX{Ob`GHB?qHwXh=|eVs9A6J+31bN$!T^(0~>NHdFdd@DtZ z7x}YZRIWrmY!Xy`?tJ`KkLw*`{x`G9_$K0O?;nlCo5V=;rpUoepzV{(2Jti@x*-(b z^sdOILvyBW$Z?|;HxEgm;{G^00wv9aJV390S};C+FrJAlgYEC1ozOHw&ybxyXFWoC z%#Oa&i-lxp-Z+6_s6&1nb{vGW)IEy!9lqzZFIc(1NAVbcYq1df30G_4%LDooMYcz| zyF)rWFSrCj?Dd0WSf>`_=}_vxd@bGBdkzXl1uAv&^UqhC<`W9~JCuf5v486dr2xc3 z8Az=E`S5Wfh^$ZdVHMkiLf5MDOJLRX1e?3<<~JQYbAVhA)}+M@f4jX>;_81zUnIEZ zs9=&rl4g>~BAy&?!nc0qSn&FZ$bM!#?kronN)2z;q?Ld~H|k z0WBMm5Itc2yomWo?#!#xg}=-BVF91Vc3M5&S2Q!$qI8=d?;G2R_7^RPG1zV5J#!P< zyASZv-kc7+cyJx?6|7}Suj$=w$T~{4GOluRPAXCMlDAAb{#UoUbR$iJ_wTSqdza&? z*If0V5Z+Ka9nslgllnuf_*?19*_*JbyP%J~#UafV!xE~TJ_sdULsrJbKtKR`qfWC6 zgmvu2U-Z)0`GLYu>jN|9=yTj;65*&! z)bWcr?_1&R%h33j)aN%Ks74l^9?g!PO@bcOh8VJB$g|(JF?iBDo8-NwjGf4p-JHop zfA!Rw^Mr{{(PBonzY?D|k|fe9h^$r2r2e{znXsYhs;aihxQbr$zfH}5rIR0oVj4-; z|A7S_00-CqS#ieI{SRV=xC;(Nmsh|Zkdc9j6p%^5TULsML>UsHmt#+)`M1>ADup06 zck}Lixqk-&ThZ$|k$ukMQ!VnYif6f^8;+q7+`7nqgUvZNh<_i8R!PNeSv@HuK?`SS zYMilmB*#8%bd3){gEw1?Hvy=?n}>#qbtwWxe{Z1dr~Lxo(Qsh#^4FsohBVNk*5VDI>3_jp8W!cGz0aVoR$jhEfSu!bHx`d zJV6=&>7k?HzH=%CN>rv3eB<()8RLq;A`R6R?g+M`U3G{b-H`LY;x%puP!yfRmj&_c zKB3eFrrHuZldj5jcubfuC0Eklgv?dK0QS8d( z+Mp^O=D>6-j0V0oF@?0t>gcPh=;*2{1Wn1%2~(_7kN>hJ0@-u3H|c9Ec8hWIdK|Ug zdt+xuzX(W{1fXpK(In}Z80wHRU*;s`lDyM~ZmU4_d>HMa69mZ7i1~i{I*&eEWs6lZ z<<{8KH6hcATgQR`;9ICK@d={k`g}<+m*x#$wHNf*%>i$}))>i(QnYOmt2YARkk2nS&1KY4;5 zzTp#($u0>OdTWq@E(fQ{FTY@0hsx`7ks=lICwza7G;SV16zlO>HMkt?E}PInfO`;} zB*W9v5-gkU<=ct|cMIOg>m)YWB2INZ(co@ge;BG;y19a$6x2*@QT?Ke8R3C8U^o$t zt3~vJYd=4(RSlHBrLn$=^-vovS#ejF%}I*INBhYq}Lpb zKO!bph!@@0JWOj9Qi$_JA-&0B;JVM@5~2X?@s0HizQ!fSI^N6?4=k7xG8c;gYH3UE z;t0A0%fr}~sE1ol#Cy9aUrJ6kJJ8!YNC<#)F=e#&^#{llQ#IWGC?8ttNCyZP2zyE= z2rXc_>UtkCfaC+SzBb6lgnP{$`nP{=1augcU29;|HFa$^SDn-3bjwwj zm8DAT(I{i)nfmlQu(T7d$uQ`?s}<634iiA{0v%?Dxz_33(}+l{acF|%GS@v*w=tJB zRVKf~ZQ;P1A|C;()n!pRtoR1D=?dTEZy_2A9$Fp z-Nu_Z+S51*e?PLgx_Ux6GOUo9nod5p)<(+Bc)VtNU|cetcGo|cW?5IqYIIygc?7uJ z#LPu$Aam(>eC4~B)R`QgwB=2_Cu5$OmYbfK@Uk37kC$TO7T|YxGMkRsiO$6)T~ANN ze=t+AQ&I3!F`5jRjOt7)zB`j<{_<12{$Xci<(ExMO+srRV`ZkM4D{f%fu>DhkD4-; zY{)f&(P9S1108{>ke@N^=DP*L_5j2vqO5PK$g02AYdp<42y{pry&U!OHmn={&Gojx z5v|zz9J9H@3T5CNkQrEacz&XVriG@RsFkR#s#OOp2cBpzX!rd6N_hgk5MS0^=3A}= zz5yYCD_U^cky%9@ifG(DQLuTT& z0_LUg$ndkyR0@iN5n;|?xi}sX)c0V%g9_zc)lKO-!Ps-=YwkCtd zJ$^PjN+-2X13aOOu^HIGL^~4`$4}qyTPRN5|7xQ3@we^<$0_~1?oU-Ga@+4*Qg_ky zWOky;{mkxIccPP`KP}Z(0H>bDr_3$G98t2yjq#Jpox`#EkF2C(lwzlG7Oy+2skAx# zg1*9vbNb0P0sWh$pZIcl-tQzk4pciQd`OYHIrt_ze; zrP2L|7iLQ=ChQZ`F;g&F%gcV+*WGsMe5Cf;JOpZl^zzU|QP3~Q0UD)Mo7lgIUR_La zX)2J!3yS%)lArD`HinvQB9Y5JO@|kOi6yY@m=#sRc-jTj9X6D}Pz(sXp+Bm}lM#P@ zEp8UEVp%%;8`1S6^nB}>3;SbnOJJn9qyF0>CPvrOVyTpA+G13dy^ zL`V)UG5L@%N0IVRhzVt8vnlgz@I*O6eONb3DIK_21bXnf>gVKs?3w$;M&| ziu3Z>j#V;y5g`Mc+B-YyI|$0GyoGrXL_7;<=+PQpaUI8wRl;dHOh}Vd#H$mW{PD(t z&rXFybtC&V?^*x)Id!aKnmNQ*ATztX{jC^fcO*nZO7hgA%Dbz4e9ILOtSV;d8L_yC z4VyVpGdah<0H2ioD<)ysE@!J}dQp;SvQt?FexeENO@Ol7*hp$R3b(wt)833; zJHM^1 zwW*AW)X7S67;UEd{y4f4l}mpVSRA*p!qj;yEyt^|=z>!U>WxbKp`V1*ou)Wy7Yvv2 z*vx`uj}}_T&RYiN?pTdKoqik!&m=P23$OF$)H^deHN;sv5ulzmZ=9;nMt8uXkssf5 zI;Tv{0ey1XY-UYDAnz*jfA^g3@L+^YPp!V#)OdAlqMyQsI5pdO=mo52n{qolbdTxC zZl+hL7ETWIK2=aoBj8Nt2bb)BHL29!AQQNv;o^HwVq{zV-chl%2cDE39*7tgqVX%1MU{>; z27iD^zGrG)1iM&~w+^3VyMMtyVzIuYtK?LRX%-@usZ^zy(GIgf(bD?<%V(>E8%i;v zPYN~hPgkfQXB}R@lPlAV3l~YYc{ALb2RwSg%=b?mfKlTM4;*P`i=end+b(KUP8F%`ap(vw$4o&}yfsEWLI zWZZMhI*(pS*OF$Nta^f2<&d#q;cB*22*A$vI&4y*NUlsEWY9c)<((9=2%%hS-Iud^ z&mV{hwe&&yU;;T%<5l_wPofMsY zN!-u^2Ds&Z#JQ}Y2mQ21UMVDTC+Ba!#&v^ZN#>8WrlqGPd@)kyn(|g08vhD@NCbg8 zTY_w`v-QLNMkj4{HRY3T+c!BS5c=78v!%2~j=W96*o~F~TaXwJvpsxVKr_?>jR~!a zTXhJ+Vt{jHxA5mdZc6+*Jw{T1LZxzQSVPO6rN#Oh@I^oG&O$4q1vofSUY6;hq`s-$ z`peb7C#UWF{vS$Ik{(PuQ1f;*9JCiDm!XSVJJS4a0H!kEDbTC17e=6A%brA?edw~3 zL>c+I761qh+~%6J9(_773v+U24Ja@S6vRf4`x&Z~*4X)5=;?q-YsR^OH3V5K~KY-;^h`K3*+W0}|? zO@a$=s)1UHZy~gSo5tVeS_k!&mH{(1ssxVf{SRa4HAM10amry8$GQ2(X~AQkboMO#MLXqGGh;fo;%Ry z$|Fr%(^WcC6Z+i+`-o_JFux7mJzc|`pa5x=T@cql->vCsf%Xo>0dO$YxBN@<7>nh> z4Po8c8Gq+Gr#1T(=ozEYz$_Nkb6nuhBz!k7TgaSS1cQY^Iq>j|V2sK*`WiHtX~$7TUL3pyEcA-B`X_Y|o`$ zq7#PO-`^%~7R-?G+yBvSHu;Itn^7zE$F>clY!Yy-9+6o{Qz` zs^1>^jk_4HWGzH(a0@EfX0HBVljndtJ&>*yVj$52zzrUr=iGIANGI(9v^*oO62-;q zl&%FC-z7GTfILkmMOu&{$Sbg=x&)w5?tBlF*M#4I#SEf)CRbsEybejf4vF_$J;&GP z{0rIkN4VTZAaa^reDU}>JThfik6JDyONx~EIj-?Q|8j`??`JBM5Yq1V-ZKwf-rp|h z8DB&sz^I(3DF~rh)}8=3u8{}}%*G#tGFPk*!y7Aw8zZ$*)-<{1TB_PI2;AATCY@|T z7uY!`{03K?9EEH4=ie{wLAb9wj)Ju^;RT6-BFWVYr|fyfYM52*VuKvcYnE!GNIS{9 zC3EQE{~uTH6sAcNbc;^gwr$(CZB5(ewBELD+qP}n?rGcBnSH+VoW1{jkyn*Bv8p07 zV`Z#Wz7)vvAXdaLxC=bbm~tP4*wKFrMy-_f0r^4*=^3cq;6|ma@X(sINz!vxf zvf>m(D5=1t0X@QYnDjWRSs-XYMXI&)DREVnM?WMPQ8_wdeyxAPS7VWDV-s@YR%;#S zlpcw0aIV=ZSp!3i^pwiu*5oOZi4XDBtIQf-V5$FRHmj@kUM+mC7kCGV(!TII?Y4eF z0qC)hH@~UnIEBlmSFBxzN%wvxo}&PhZpxW1WaZNn4p?eJiW*`W={Vl>4KuRY)mh+? z@r>w}+z;;$mIRGKgQARpeWb~(8(=k4iez12ZEaxT?4Y3=rW(lmDB`>%GSc+RJpl}P z^5hs)cCB8ru5aDEpnP5D1ogc@YU8j=0Kvg~zVr#igs=S<*>8%D5lD0v|76R_7lYS~ zc#b-I?@>cHCig$j;)M^8iuwZ=L#A+aV2)#cYFi#_Mx=kWh)4#&!@%-Nt=c#QwJ;;P zq*eSNo^1b`$7;0^#YpCG)?T)yOPaI9|80hY)Bi)Lhs*%@iZ<4%Sup5pO${(e1jO~M z-hA}$$l!`k@=h*hvKFN#`r%OXy4T#zqX4+;P7;>Q9Dr37)vLv*D1p*zK+8z&u!}II z7S)lA581nu-Ol*1nK#21oRiOcN^h~gM^FD1I`eJAMH_xkTtK8jgH!=CMTa^#OWx2?of^M%kLFJ6E~uXo z;S}7>g+7Ome%N?x>E<}(rz}i7Dg!sCKZ~T!69kk+397E_&f_`-hR8D&X@}1^B}~wZ z03S2%_I=_lTRG7s|HyMfnEsTjv{%Gju11GObj$z6neQF^AgXa26;r@ z_VqM-XAQ8Hb$JHHc@T`F1$R*}HMs-^c+{$#H*#xQKgW%f$S0Fo!3kd+%te&)S8 z=M9?*DbmS%)Zn=rfvMj4PjpPgn@K>U#&w0fDaUKCecCg8UfB=Y`p@JH^-uD#(w_LR zvQ#A9ll}tpJ}EF+z;47akGD$Cy*TX(??j#PQKp12xE0Co;eVM)Cy?i;hH%eL4e_xfzDNE@p)0T&rx5amhy#X1mQwead z_nDpVYSHkwyYjvU;QV$%aGrHH zZBb6t7Hn$PE}imW6G{M4mG`eq49WX|2^?OK9>L=2>h#u7wus&1Dex{?shm9uYQ+1;l-RJ#%Rex=KQ;s_D@F)?fm?WAkcQO z)>0dGz)O4(l5fcBGNzNL{Q!bInY1UIJ3x9TO%o9~2aw%vZG*^?N@cnx+j%BbVydV>qy$vZ6_6>@&J& z&wKMO41A{Sh7Llkn#dm@#N56+UAX~7Bfj+^!`R~NTU7~bPa!&20$#fl~bOz{|mHM+f z5rp@s5yjaoOEE97ghr_*gcLiRPP-N*;)vixdDS)ZM z04~aT>P{!Z@7>EH=VLPN{!)e$?ei}vyGaNe%npf|+)z>O#vfOLvD>0}99-_Xq$R31 z)mB&a+_3N<9wPFVVBE=eN}k#yaq&dMMCxBWT}5QdT_ttM;wi#vb_@`0cehkOYpoeuOl~S;D3SPl4>gv-wGW?`8X&zpxm@ziz(0W{^) zRt7Q)ri3v=#&?agIa-eK{7|9Kt2ZSDH9V;V3GCmQAAaPi#R3wu6;j2X{mNCvr-*vO4 z70lEZin8?b;$Q@er{*M`C~l870Xsv)(M7BfMT)Y~T;0dK1#^h3_|b1pS8wY9&2^5I znxb4E#2)42{RkFowiSrKGx@~5Ud%E4x%YM(QbO8fc^S=ZXU^puGlg}SSPp%g zUc^gtw|#-21ojyfm_|GsQs!&xQlXVopsc1^F-q43r|5DM4tY7??a5X=Fg1IAkg zMw_{$^AE$Zb_fuuXp#^_*mFf1g9x@y1Ncns+3@=3 zWB#yqGE>qUyD?Eb0f{*$#^nA^z4r&4#8xoTPW^pduXQe32lrUcjCPhFLnFzA*&|%% z&3GNnE0GPetcENIUy&0n99s)qa**kPL%qp$NW&h0BO!79S9yX7tSx&~RV-E2hj2jB&OGx- zxHki#IV6)V##JU;lcId!A4Q(n=(a<%#vvG)&nasGp!ywL8l8jBpdgAqKvVZ3Ycd3s zm2&ebJk4*d?ynkXjb!q5l*MSv$x1Y(`1HTWwh4B-Pw2NjHNAhpTCE18O;vY`dn4`n zG%GJV;qy&DD<{6r5FxDQEMUfYOcbT7Eo3f{af5z`cGJ$Cb3L)d%v(T5tQ^l8KqQr| zt&5`orZr6IIX=QF97SF-wpF(1)abk>Clq?STh9np1J!I*>{n4u1H&e&Ra2J1Dl@U~ zu{mJW`;Uh-3{oarC~PT=3SMHme)5jq>`-^3-4bBs1tI=?LHGdNXC#s`Y>a0tdYJWi zB{T}CRLxPiF!?ELKxLW>iy^4q#6OsbDIh)p_NHgTn%PaZ?a!hU-|M#kk2(JBuUYfPaza!YcVvu|uWp!~#-7C= z1Nx%J&bk|5tDZhX_5#nHRhgP0BmQqT{OH_tkLzUyMb`L_!g&^8-4LIyr_D`idkv@cDR{NrCz#g_~<~WOS}axHpV{@ zo{?sEQIX^y)_S+iUW`QtQx!fkUL7BRPhjiR(U#usv*Pxh_P8wz9C=|jY z&coR038PHfpRU{WXk=vfutc(KOIMwr{1bvukSl+yz?|HmT$M}3B8&lX9~US`Ez{+V zQ|4Q&am}B>KsZA8HfwY{*iIWJOj&&CP}v{D9|K~9JQ_m*}uhra+n}O zSPVxW*SdPDYTxePI5r7G3~G40;&C#dN*%J4+e&Rwq@%O(7v;PnaWEUgIF&!d$zx;I zjqh^L>a6|R+q;fare?L{#!gFk<>gMDR+TUQ7+u=20cn^FFS+ghC0))_0$mr03&s{s ztH+Z{?(vwmhiVoVOX}Sci(0<`j;h+nGHJ+%gEon$CSK(L3{@q6;#A2rqV29O^A9-I zdK58S6hW?~QuZTIXSzx~)g^<5DjmEzsW%^uUUZkf9^eyrt4MmOT!V7Aq~%U-?u6&r z0g?m>ia90s{p1(u3K=Pg4;je{j=XkXHw6R;TVm*A8M|VZE+OiI+>;hSLH#lFLYwXU zzKR3cp=EM;`0JnVB8X1VNAoSQHsSQDWD&>5k!>9XA3y*rWo1azQ@#pRHn|IRCH0s= zujMy%-hwNeTE0@l0xI^Y!q^So>rXq*^hHcZ%(p|xiPj%Y^dNFb@UCezx*R*^lW$oH zX>`dI#2J~;?VPw4cWxeljcW_H^%wsm@UA*aPQ;-3Csy8s(#aT##(}CDGum=_=n$6k zxCzOPP4T|VSM&zVHF+e&=hAo6N*mz3zxAahe!{_XkgtujkkJyxw}=&Lp`2+I<{7G> zrWei&{xxvO|Iv5n)mdSks(_qpirkFxrdqY}-dv(eUd$G~s`xJ;b>E2Bq`az9?UzI) z!>ZoXH4D%9JHqm>uL~Y(!lRvc$5%(@$D@F`q&k{LvMWkXNUSaw>}^gtiD&xG*uMsz zyrsUU;uYOWdZ#m5`pXT@WVQ})+`Hwysy-!XclnvznPzIdj57s}LD7NP>52CoX7t-V z(ULD^u^Q6c@C#o6g1}QjmN)cX^ zzBdq*FD0Y1HNC+#aRoEK$?uhy8*~vvBRiNBbykryuoyG}&^)PBLw{}JqM zVd*3``z_cqGLcS%^MK0aCA{gW%w2P6?I?UR4BBXom$4hX;SasEZJxy)LW^DP+x#;L?PnP%Hv&>}&4wz~qrZCM_WHm;X8)E|kjc87LeravH# zCV`(>qp=g8InI|~Q}&#c=j8T3@qC>zZqK-t)l@XS-j=gib)gF5VARTw#T8lA?QiR% zyeXVR{h!#DV&IC-twYDwa8?j_3EAX2VjqS&WLVLk6@CTPeK7KM@$<7rl2W>Otz3qe zCf^J8WSgmh6+er_D|0p7n>ve28`AWS6`x$m0}Ri>N)Cx2h^aeSZ zk}GKI_jsFKWQwp(2_?{QXz2vMkGb^dlhF45R?i3p^dm96hJH@4o75SOk*EdE3MiZ6 zFoSOUe)a;10uPA%cmIF2`L6;0|8fIOkhA~Se-U!?hQ{TVK~Cu6F9lg1tS+M6T0S>v z$I67a0I8H0`u)KLbzul`i*nL@2O*@CoP2JM?VX{t#JE_avCzk=AN|WSM5b19sO)mM zD{D&!wwheX(c;OA=NDX0OzpJJlO`^_+PjO@Gg2usfV?^m1LS( z*wSGrXwk;br8)ip3%}}SP?E8`V{W;;EW5PF>I(#N)vN4Tm3w$1>X_FJGXS`h4R5 zzz*I9;sEC4O6}POqT@CU7nGDF)JY5`&DKx2w%I=OdGp%sOuQz|E?$%hC5?*~BqoLp z3m~Qr;O1{F4ZgRcopiTv+n?siP9@s`A^;?|o8H7!)3BSG*ys`T-Zh2QZ?&@(>Ix=; zX3rF)lFrGXoJl0K5!lMSOGppcLk2WI`M-W%eB8=T?GWSwVoN`w5s<@%wf)ff(n^8_ zi4=7GYOR#Cg3BZ|)#>!R-|y5aHEOqqYHLtZl9M79Q_<)Y3?2>>FVstF^}6539|6(H zyoOpzLTaAk{%uJbT7)A8)C&fC7|C}-u4j`u^k|Wl9RGSvMTxc46sfd_F;aLNy-hoc zWOXxg#!>Eqn!El!PJdE6zRv_m>%Z&rRURM9eNLQXGbyCy^fMeoRWd$)D2G9Wx%{PC zaU%{~Q#eEst3ocI%}*dBw1Tn(?guQ?KhlEKr@g|}CuoXQ8DoW?gaasA5I4iFhTx4T z>LbTU8^UaeLpddR19B-6!d|pWf}%k620EprqzLFZGd>-h4hCFdc7;q&FT85Dy}ahb zlSHU)=y>@+?_>;c_)_3Lkkn($C+DMX!2Hquv-xW`mMc--OnLh+L?W(1^mNNs*2UN)yXpsJz@%%2^fkQZ zb|fEps|y$55x^2eIBqg6>Hsvn^<2k8EkT{elG=nlmN(k#N!75$yK#VGZ)w6WIR)s} z#B!G;t_^trisAg9Rpc|m<;fQCs_HraopJ1~q3vHw{kqd7wdmL!j^Sk3sSz zq(f)MpZv$qPoL!=W}|^e9ePgs~+o1qBY(_%E%#lex@R_vX zNM-XDrnjr55lA`lvJ!Um(Ua3C;UXeKD5Vo5D8KdC@%UUlEN zpXhhQ#_R{5G7iPb1Ks;7x-pOz2q7ulmXFSVP2ZPEc+n1HwJwKpE4;_GBd> zT+xekuqm@+Dm!XdYWkvRRpQ7EG^rvtu(i5sDi#uvQVvYPNPwz*^x#Kp<`Q3R#0WyF z^y~5`BA|?AkHb;LyvD}9RKUjAhx?&&DBXKw^m#+i?UpEafbQVD{}=PqW#|-BjBeqQ zAFK&|w1&v2{p*rh-LL!hZGKwNVm-TDzUbsQ&l3#9!kZDz$6sHRn-(h_{h-I{wVRF) zykc8n%?d9F%>V_>Z1P^H0P)a~8gP=^Co4g*@h3viVkx*`vrn~cP-WyO@;`m2tSy*= z@Nfy9fl9^4r%&A3>H?h!T+{7RdaiI8diFxvnHa+4g|+@k0b_)01$g5$g1`HJ3T;6x z6EU=nyXOQ0W-OK_(+p6A)K?C$Hs}IrWd@5BoF#{~W>hkP9{d4Oo ztD)HxQ}~I?aPlTh9*4T^YCl`gR(cZ)ui)d};yCNZw^X-3tdyqOXwJqh>5Vzhn0n$k znhW0keNC$nM3wlJ%1~n9YTlu%ep3aYMb=ryB>$RIq%6^Pa=ucGZ~&D*t$*B?g6ye%u?cN@3c6xZNk;rIrSa_QOy_Z>xDEGYx00-jQn>a{!Rf zj9F;b$s57E5F5^Pooe8zAB_fKxaFgT41>6r4k2!+<|y0ql#;Ep!l)-^WU&H?Z zA+>nX2|SJdC4i2?B@9Rn8vrN0JR@EVIrw}~r_FQDPF42%PSJuXFyLbRA$Df> zBufuS8F+?zqif5Lz+1X~G4-Lv@>m+MuC5*0YI#mJtGHdE>8!^}As)`Vn}<@EXAw^; zRr{WZ~#%&wobnIzOS^Ehr2GNRV{H+rOF(}0=G(!q#=;%NR zleT=MF5Ce<{O+-xql=3#{D4uk=V!w@6|qtH&wM|r)UUj4QM>HRaNzsSW^ZKF5pppy zGaC}KsBa5+l><8ID4u(4Sn4HAWHsnlU`xRbtg9FoXRMuVfqESGW1CGVxH${9w0GbY ze23|HWi>k56F8_@M+V?zM%-d${=#bqTKz^8;M2?f1FsYl@6bUIazG>rqK~nsC$wT` zD-=zaR4l{@Q*bfgv(|5Ux25(A20*%`+~?%PV6~*qC~bB(?`~go+0HKNUwFU9-E;R? zH^2oJUu8RSun84#newd%_@UN@2HV%(x0UilRLBGv4 z{up;wdOL;6+E(6(+jnaFwE47+$DkWs{6z6e;|WOkuMOig@ zENHy=mbWBb*=|9(-PZ8@N+k```HxE7x=tT$622;=14MGc6hP{G?TP?Os;S3s6l5vS zDO3uQhi6OAf8WuHsEe0igk8jJJRf%1Yco(Q4uS{g*-qJj6LUOR8F6JLHQWh}i<9P`Bpx70u^c+IRoW~ZUPLsJSGNwm}P$p?BVUoK(_=%_!kVn4VHj%e#Y z1Zs0?Q{R})Mu3`xHhcBO%60O}ad>tcM|rXk4b81S`|Gl-vDbJu`F=_Jrp>b2-}p<* z9M?bx@FWQbFeMh6gO#(A!nTQMmlTe^G6k&)IjCc$;}%vIS_AfKE+yMN)sd0qyC*?5}l~yZO#d06g&<;$5e%e1$2vH-`y)y9JZbKb{e2C!}p;@c3CNjr8e0~#Rc!i`*>x18^c=Is|Hbg&>qa);f;M;Tf zt(EXLnf0?0#}?j4lWREPE`IvFKm2=wddQu)F96SaC2~fKWU`G7y{HW1r4xG(ct!cI zJy56<2!6O+cXut_w8_TMLV+JM#Y*JL3+U7(0zciRPuqTj_C&_H9V|OfA*CrEO?^#L zPSB^mM#%#6sud_3V@HJ=s_NNsJQ~l$4>m;@fiUGrCbQxKA8KK-T|=dxXC^O)v4gim zIza5j`CHDlOf9ZJBuNB~Zv^UZR!VG|dn;8}La=s~1@hAbP)877YA?BFqzoNNjgr|{o_e01DV~|}jFxQVt~nz$=Ekko zY{}<*X4U@98kHK|6|I|enJZO}oVb?Qg>=(4)oTd}6>gZ9o=dS+oNH;-K@|GkV>2#xt znTMndVXyb6l#>1cz^?nE+h<%cMW&MT2m51A4^0NI{)FDmYQ3d0plv7^-sWL z#M1d?pBLrBS%~N+w4+;!8F8E7j?GKQZ;`+T!E2~SLwM8&IKA2tIt_gQJw!s$S{aoj zhmyQ@7yhP1ky3m0A=023dnfacQEza?IXMAFGl!`?Ap$B<^hXCG0Sfty&qDO0sjIx`H?=h zPfJ(+g*`3?F!+nCs`g^|M$y9mf&_m)z96?J9gxz3F2GSmz|a;97B&Dzg88eHC?oLT z)-fy6-2wLmtab`5pza*Ao@wcBQvpy=1f}W-MR)bj4upN;aFL|3d9^e(X3ksPMbcl% zMWS(XR7ghH7&4}k*qsa^&cG&Fv-SAcJk5%ii@ggOU@||A%sOlj6Y>dUA>+qxaIug0 zowHkgxQt8+bm+1>Y(EK3ilfx+aa=R>_d$((oMd-qJ&j4K%>apA>rUUncD4NJQA3b1k&brD=+m*@>%#BV&7JQP#tt@-{Ox+e8P`x0P=on z2O`JL=iHxOlf2^@ufa@6t{90yBk)wI%e{v`2p?Szkl1f1ES{!KJnuuxzwmGQ8YTI^ zc(Arc!!YC*ILJv}$b%O@($AR0_$=tXp)opC0Z95018Y&VudGlkEqirVA*nrcx-sJBdi%bnsSo$T)}=MV1CX%Xvz*}B ztRgc=n0_dDU;8@__T~z!uVlqr^E5;NlUf&-ozu_w-@3Se{;EBQE+4n0&IOVIfJkIU zul`$AaPI#3g>w84$z_6`bs!^Z$juj8`z5AGbn}&9kU9mVwMK>MC9_;iHluM^$~-`T zZ?vV(FXZIOyX$yJ?w)nkb#|-U))a`Z(yE%)y+#{d4Bq(K zkaFkgdBV(kIcLvuiN6Y1=Yk~@FA+HTuhA3&Yo6CJmt)kCcLRkEST+)J)ttElvArf! zWP{-_yv5VaM?!;q@+UfaA%{rQ$(>qM5@I(onCQOfKLf3zc8Mfp#{yW(1B4-cJ^?`p zm0GWp(0SOz&m>Rsw8;}un5|>}@Jb!X2%_*Ld{9zlPsD^aa)GQZrRg|g3{@6bdjUo} z8U8twi1W zqS;BSs9-Fr;3h0e3X34JMIB3+UPG`dUrS*E`RQp9g2ns1yWZx>WuUrnt}Y5Zv|_WL z6JkiN<2+k;3WA3ym(1{w+p<;LdK-h4nt)+3uo0D%U!9d$)|=lIWR{Rnpz1asAvo7tAPoTeLuB?*PIAUCGh^^kwe6Wo&;$s)rGvfl#4{PuRtYB8{`f2i9KGg4ipzrh{7a*#}!oWt&*q8kI*ufA@c=Xv39&z z2Qy<$+vwq3Rp=KKQ6o1Iu%a1Iv7?8V;ZJ;1o7V$^fge2c%bJ>o2vKpOi-nW*T(qW3 z)bnS8=nA%j78i(@c&5~mZZM!H=rDOE+3g-ZyMxKSbn*d%%ewVDgdWvv#Fwa>qNx%# zs8+bunrmhak;2D^1F#i%ZlySp^!YRMlpWeg1*$l4N`Z0cXNvTmxJN{MMyg@L*pJxn zF%yNK)Y%nE&nHDC4}*uBj!s;n;|kH0M;hT83N!QlkfMclaUMswQgcFKzhUjdRgk3v z;L~ti?bMl(* z+<@)dNHcuJriX8o>6T60HQ%FU_oQ`zn91peG2Uo)gfYG)AUI+%wo8DB>>Vu+7YhKW zB&r}r+P2qw%wpVt4N|--7_7nZ+Z~=i?rHSxC?uuc z#`OYzvY}qi)C$N;e>ylt z^>hfme$*2Y#`M0n-teqZwmezW=_OT2L5K}>}Na|- zuZkTTrS^p4Tt*j|^eDCzHdxjfpNdj45A*|0d-6VR)5kCT75wlruCylyX{yP~2X_KG zTk))J&d&Vbu5odsm0 z;kL}Tf=Cdi^8&Xnf-$M=-{2hCf1vkGB7F8$oNQmd(;ED4AZ9=<`OqvA*ZTdYQ3{w-GZ@iWW#Em5BW0kZWxBAxilA&OST$4@+sH&`PjLDM0T?{?Jnd{k{5_>a6wJVg1`&gpgW8pDTFaaO6@c!Fk=z*?fWPi0tOj zhY4g!$={md3`>Kn-*jnu6HU{sz6Sgl$K=1AmmQQ1SWw z3!~RI<-eHUUV-Gn!=N4Dp%z=^-hjq|K&wYuG5>o;_W?wi8Vv%B2>4%e-|iC{?@LxW z#FDF^I7Boxc$YE^y3dC47ud z&v3@QBDj1oO1eR%Vp{Wk$rYCYz2T@hy>}XG`dYkNo{Zk@_Y~iNfjglyj!;EhHl7Dt zC{7j@ZAuy7xUWkTlhi#9&gBXCSLAwz+Bs{M{py{*fk1exol+gQ|9R2Ae*wt=@Le(+Hgk}~ zt(E-=@jpL^Gx#2Vbz$vvb#X{(8CdaoO7dsL7Q%a)!_CGg>8><%yrgQtvn+ zr;wVG*wBy%6@9%QUp=+&-^fWDT(?sDfCYcaw^ySR@tnU8tgauZUhMob1|@uhjc9bI z$=XReyS2Q~%NFK12a~)yq5lmjp{WmMmNM@ww8jOKwr~rYb;B8p4dJD-=_Ft@Io!TSK~F)xEgOwZ@X<9}^!*e7#Ax1fCJ2&~xf^5hd64d*x31%AdBH z{bYHByx*j3@j+L7+RhH3uWv`5TDW}d2qvCAuBfuzhob2QDPIsNGVhy; zV`EcUQ0hWd#Al*%s6nP&oZprozofN!AeeJf?h<5cCt^)MOFm0+Lu>cY%+!Nl5CLAj zwgql|J=Lwt5E_NWTFI=y3E)FA-S9VDcaM4xO_japc0?uk;JzI|9wY->M?f!)csR!TQst83`FC39!?Q-cZ`IiCKg3ehwryM+;0IB@0sCa?WK-}h;AwY>ZM z>^gYy^Z<&8N=)ii}YaRVnK= z`GZ|rbW6Zp&5=T%GVo46dVb;~iU0m*dx^(n9D)1sjkr-L9pSx1U2B`#DmyNB-xbwq zB88uza=oi#%1iBs^uSh#S7q19!p)nK1`*P`8)EaQ{k0C;0z4yoZmvMvnF=W2xp9tw z2eMI5Nh+)Uomk-h@YdlK05r}4dea|NSZln)4+a#bXTHecQ3>u_<}K>x8>O6cJY_c3 zuW|akHC?;zb$c;CuWJz5>Vv6>2rL9QqbjjH!JU8LBvCpzzhDN8a%)ui^GSHaA#Ok^ zrz6>7nMe5uK0__shPV4$7sM*&d!#&$g3yzf@~G8)g@AcbC5Q2G;CyCN!t6QN8GR3G z3%8@uWqL80n$-oX4@$er<3efM^&>Q;J;U-#4KHahenPttj!bK01zKF~@G_4(3vq|i zoq}Baii=$+ohJn|79;lup@w|e?!Tlr1p%f6F#VrNucz2=H%JKE{eZ^*uhBq~;wgw& z&XVOQ&X)QN0?|Ra2n>-Z^z#ci+H(zvM4;FFBVB#Ey`CKS3rZg9f9{63|Nm}iu`K;V zMgjV-6?LfCr+VY!>}q9s7>X4hLAcd3=pYBcVO~FjR5}6=&q?v?vEwPzSabh2UoVyc z>H{7-u$~0hE_p0Wv+L)7d6kM!Ft^g+<4~14J}Vo>xrq*7(fF^rO3j@En)T)p&gnt+ zZ-z%qY(9|PB@(IIy)sl}AUC(X11>4yIM8!7?q?{}d;yPEHl6?bsQ%C7;;IPDe+b?| z{6moCe;}BkU>zukIDF$9!B>*Bmz%bYDk|B8o^L553x73zY7yKvQ4#9>ao$4~3Ju-6 z{m-sEtG!Hm1Dt)Pn~_(LJ|%AA2S+@YY-%19Fb6=%euEJ;c=rv>_mWjDh_fX!D2@^}+ETA{-fEVJiB?&v z;1|Nz3kX2MN0T(Jhx3{5WUc4m$gJ5ivk~hDyTb}+)@--w2~lC&w5(u#SY$DVyuCd0 zoL2g0x&Bh(;K+IXhcZhwpzX$N~e(=_A-FfzP-EhIe(fIU~9&4oY!+~ESyLnaFd*txpeedzbI~g&~ zJI@@C!AIt#FvqRrcMqUd>hUA5N)Sp>a~fm0L4uxx_RI?yDTy0Fud*a^@kHXTuBe@B z2vXETMXca6DP9!fGA35tgedcyGr~g~8~$5%ix|@dDWPutcdVj=7Zh?ig}2&8xm&SA zm7&NJTB;e6E(a8*36)eSZ#c3@tjI-8XhO6|C$CDd6eGSy%@v?}av34=^khn`>V;H6 z2gTjsmW3g)B2-i*HL%Z6nlor1-vJNcn!x>bN^p2}t_bZVT==u^&5e6>88H}dJas@B z=5Xk6;fN~A&8Pap%~y5iXu?ILA2T+50q!8oiyr`qppJI?yEvH7iFE(ptd7tc;r}w9QUkdatv1^GH_r>JtMq>~Yq?`(19SS$&+PI??nK zgx}^hjQ=mpq1|x6{|3mv0s3!%wRXdSQ&RyL{|7tQtfS?;*|y-*+Z$ldQJw_x@w}{B z(PeTsBtx#A5?+gM64~M^k;0mg#rd)dC5y_S#t8^4i)eFS`yYRt2`gtA1`p=V6{VpGVR<^@nf?dTRo$dd&_inbum1}XOdJ}c z8HFf$d!Gf`47Of@4j|lr4@#n6rBzVI3Y#-d1({gMfEt-dmb}@219&=`RF4-Le?khf z#o`E5(5(!zS~0I5*f z{nrRg9TJuZ4dHpxc+;?77Bpkz5J}kF4zRRsnV>r_bX^4tF6**#ht(R;#p#$ea1RaHJeQ$=|3J8EW-7P95vHug+5gs0sUW_>N9qLMJe zP>3WjqR62$fq-UF*aA|}KqNTqQYdq!E$;1 zFk?e73D}Af!0d9e`jLjm9g{}WBnhU+nIwVal7eR(m;vu4+8ahwGb+uLVeo@&RFK_p zyr2poZ%IVBW1X};fS+tv@CU>14>e!cBhSx)!|#b>{|nwx#5Ru^H>H!PZfy6F%RMQ3 z=5dLy{x5qkgr2?sl>)z~R1I6)$ z;fIr-0Klbi7?UYc4XNm@>YW}x&aNE816CI!|Fw7KVH_Zvfv0~cc_aF`Ehl?&&CdBYAR? zEmS!3^c9D&k=2z(+-VspdXft>+%L;j<*F7oKfvY6uX4~hdXT1-rJ9Tg<;AszI&8Xx zvxO>EIJgO{dMPdqwVB%tryj-Yts?wHXOj{r;YOqZ#fxHSBD&Q`{S0w?F`4$1)x!w4 z5=R>EXxaDbHefUe#_0w8JVt>y2Qi2OoT`hgU}hoZ{rslf&YK3YOp(1(#G64@?muV=GVZbs-I zt-w)^{(~!WN4ANNW2uGEmMb3`4-6~ODDMoAF6+yVw9}qg(EvW-jN>`lbw-&)dGw59 z;C>ESa8xRdAsGEkxF8E*ZHHa$yiNQ{BR~OqxWwl4lU;wm3p`k**h2^PkhqDJbLbkS zkbOelIr0)XR82GJCcLbYHiuY87?y(ZOM?Bm{%R)z4ry5na?vLe7;-ZegY~s<;x3HJ zjV}slelzF~2y_uAn^~;RY;^&llTZj)No2k)ZaCdar6R57Fsst*x{;rrThtm5^Z|$UQTy{)gvb++@p61{mRP7y@ss3(H)Y>|^T?iktf7E(6Z_$w`t~^P^E3H- zp|_)Z_WJ=8s&s!W_xs_|(uxl=7T)vWbG-j&KmJH)PN4FW<@#s(tv9zMMn2x_QeXK9 zwT{R&&Am{0^uTv!XZC)v^UZ-V01!jby@{z%QcU3@f7Qwfa;wX!@O^-3P@;cqhA+E z{?G>WOj8U~;@D8qjTjUAxXLm^;Uu38JcAdqGhs{NGBL$1I6gBw`ZqEWEtxHcE*G#7 zyO8Q)oN{ll73dLdfEK|~%4)eWG3OZ(WvRcV7w2)>+fMd<*Z5Iaf3@3E9OB z>CO#iOdMThsMytVc}K^GUBlDKEFT!ZELt5OTANN``M^avUAJsi9J=d8?Uw2KD6ktu zI+?W!Saphfy13Tot7mS#zEJ0X>1>O$FW2cVvCy3S)YA=&7$%FuGq&qZyrPQYTq*Ks z)fMI(5djr5I!oufGMv*%*>i&2BCp5M)tqpTzD&32p!PtNZqx4n;x_eV)HyYz+Vptw zD4>7KNp9Cl8UvvnPd<*#@pOvgj`E$Q!NhU7I4;^ICsm>XaWV4UkVFQ5wNOw38wmo4 zbkgyp6`itJbny=HKqem0Da4aqRU;0_WO$hjPp8z@C)qy9_Nl{=8iaI(xurOp;vANu z5&EZlA_*v%TRc^KQOuUNk__eRPc|K09oke~sx4~TFDp3Sw##h~vRK=y(Lc5j9sNWW zenMxtRHMugQ=&!-S}ZAl!P_#UPKgQ?)@V>4b$6LTyhNVE&h9SeF=ReGplf;{*Kd^#9iKRN6p;Tg=oJw>mR$uPgA{D(h(&JI4tkeMWZb;hp_4~}~iwy0c6 zd8+-h-Pt7x@Fp<2OXVG*1v|q~K4jh@4VYlq*Egxo?w9M2gFX@(1jJF)9wkCMNR4)c zp?6J|s&&~Ra)-g??&x}$XQ(*rT_d}I&~L*W19{sUjc57qoc>8JI1(HjV)1!SZ&Dze za+Vr;1XGU0FKys|##%22!yMC?<+Yv3ie=~y-(->(qswt0*e>DPd0l|f??L`J?j@(( zuQPr>8lZMgZ}J}Mu$c<)KmYmZwDat&b2c7b&j&ka`#Ui7-LN-Z<3+I>{dm0i7~ggl zGJk*}mX{gAn;2q)VV{b!)^^1f&0U!-3Ur~DsIav`TxPd_=L_e0179A1ctIYk*2tr` zAkPCmv|*B4XOe5lq|zY%>iqHf&4*`4$H(K*SjdWwKgX9fBBl);&`A2kqNd@l69xgn ziZD{jf-u&U2t##NBnjmyg(8g|ML~*)udX*tDz1?v^bZLUFa4{KWD|M|O3=%ByY#=v z8NUTXHi+MUDY2prQ^Gn^{PuIsVdwRWvriSe?v022bvuX0pKu(1^a)2)2-J4Q(Hvx= zm;4-teB>Et6Tl1Vq}N^L=_bATYDuO|;@dDN?bTN)6t<_(Y3J?xlcQ&y!EHV}$-llE z-}Xjt&*CtSJ8ML8>hXwV3Lfbn=aF@;G!F^e%p+8PT5(5~xnmHafjcgfv>}P)VFP)j zmBR2f1|hTrgXmW>kbyE?DXyzA&V%?ad1TgjWVeZV9_*wQo75Vc5LKYDfbbyVXu-n1tZpJ1c_P9^o<+ZtsTk z!DRT2O@?=~RAZW74fFfi-Nh`Qvtd4)<^SCEhU}X5{?R60^d^((_`WDlr{k|v(uxg6 zSA!7@#PZ}SY?NQe7Y4IoZ+63Odz0CG%<_AGI2&0KjrZ7}>>b)c@+^<93)p2o7!C-5 ztNEgwq+d*XmwCJ$OT|qrjX3%jLpJU82baC!(-nCu)_Rx;W#XhsV#y6%$Q5nZZTXx0gZ7Vue@{Lm@zsMr6TzBqY4b3sdFAK|Y*+ z_t@zzOvNVj&MHD2O9^7421^$FUp^hP@hE5WuVXg9ndUjW8sAOX)!<9cW`lb+%fI9! z7Byj`0V-aOhp=g43YqQie7-ax%CU!%#mXSIH<+y&%x!jNTQb=0%MRwR$PT}5F!Re? z**3YdLKF~;pKUZ&2LKnJD%jDi@V{l;=K!pg!{)I1zu9n zSHiuFMSI*s1^4u8x3Z(i(kv=VvuNkV`!}bD=rNdP57vwQ%5nXc` zC93#e?J#7f3P>~h#j*NcFNvjnml72_MUxUA6buGuuIlwa6fj~M6dy9+sh^}ULumFV#^Zb5Zi5udL<>UTfHW~K5Rcb8m-Sp;P2Ysx6VS`~mr&7sC z6(qOIM9%jAIz0MV4`Q2=5cl{=i2sQs1VKzg5>neeQ|0_slyeYU%|WtsCXHNjl_u3ai}rla6wAU!7G$&Ed6LdAuocZTIb~&u1M9 zlN#pK@T)>iXPZJzyI44X-TG{*dDZD-D7hhsEJq_N^;$!zMFmPN>kE`{<($r~z^mc8{CT6OX_6@O2tu4$_wXCWrpgh;vnist(h4yf{ zDqOw>mo4GtOMsnx_;_}5ir)4b0h8_NW_|{;9b_9jTJlUPa4ECfeAzG-S( zFPQ8qY+Gv4E)Fz*Yr!@b8zRBum!~g|Kg9UOuS29C-4|VBAm}I%q-2(U^yHkVf>VpI zo5HZN8r!{mdjEOvuUGFD?>G1_eEF3A2%_}z$*Z4w`4l$2e5$_B*6x0HUHx0KD&x8;f8dYveCd!oGmC|M0;eSgc?zT zXh#%ZN#WdT;VKcutIb5A_t|_w6crwLizGI(#XmH!G$UAIOW+l@gl%3!HcQ{0Eq0wP z3rGEbU~~prV3c1MCF_8`vmzQIMh{R;<`jv10|q-n(K0v0(!X_DP({W5$^z zlRRZIlP41s$4N5g?)UtC`OKN$p1XVR?%uQeH^!JRUtGp@$HZbkbm3SWjT3Mjrr|_P z$H|z98JK^CIXE42F%M_se4LB~!5BiaOibB3+tZRHRLrRb zHfco?hAT)~i>X>d2klQtyn#eDO%r!7gOnD0;8q#a2|T!X8~!!Zma(6gVP zPuWjJ(%A{uVIgk7^~%a!U54ej9}lv%EA?U2&f|CjPvR*$k7GBI zd-wn!k)OoVcoxrLcRY_5@S^8ar5vl&9&0Cg9rnPUY@LcxIDpQT)aR(5(%Fl;9$(-q zd_$+8H|js4H?VJ74bttw$J?>CU#NjxC{z23uaTF$DGNxcE#?wETdLHKELR^H4aS7@S9D+kJfxLeJ zm*H~M@2~@R)2ZLH1WR!*?!yCk2#?S|jQSYrH@u0r@h;Y&URgZ@53v^O@G(BYA>3nK z=+|>Kk|YMZVR!6-`cLUa=O}8u!}Y%H%RUpR$KW`eKz|bTM4W_aI2os4I%d$XS7#W> zLR^eXu>hCha{5y+l|=u!tFaK*u~mO($aLz_)NQdHhG95HUUEFBI2?b7gHi9& zp_o8_26ZCp9WsKw&7sc3EX<}~XL}A#!|6B!b8#llqCXS$dd(%@j9YLk7UK>&v#EFC z9xTPZs87K8bZ*BS)M>DQ`VsX8+=QD^C%@h&TX7o}<96JId+2M$8eYdr ztfEuz;Tu?uw@{xQPx%#1P~U&P^jx*VHW-Ex*a16ZH2rzhU8(D-hoep#og_LRlIhW_ zFq($p!X|3089?r*n)N`&|7vU0;SJeCPARfkJY}Ngoz>|0i zPvaTXw~^PVS3Do}h7C*7g< zfO;&QqDRAh5fdS>)VSBhm=j{ET-d+SJDfS2(K>b=()qfy`M^f{tW zNqu(oN1e0{(HH%&5&il+^~V4V#3rayr5TQ;U*F+Zl5|0RU+sZ?Fc$S~MCYx(V~j(c z$vSCue(F@zDW{V;9rb_y^v7lxjIFUP{cEVhF%mnXK`(TpFa3qMmLvd!Fa%fND%3se zb739zdMv^XxDhwea}BTKdaOj92{*7BZ=$}7>T^N&*%+H+OAN(!sQ0QqJvZP+)T3_} zDad!$s)auGw^y*T(6{Kzcm=QGExd-;u@bBB23F%uti{`S2k(Dk4c^21_y8Z`BYcK+ z_!ys{PQsU{bL=_l6w>MOnqQ&6NY@2##T{E=*W-QBAKj>HZCw}Zs#({MjZxROx+2!~ zudY~wFc@_Os_Rx=!RdN40(F(AE5tt76?Ul;RoK1T@z^`Z6R#Z42HEf2U*z0M zC#h^?lK~=CE1Z9%=Ce)WMDCAvlG?U5i5IE+)=7T&V3UC&Kb6>ekjU>_ovk0j9shV> z^T8s2{^cZ}zO%^?5zj$9$v=HJSX|@i%k+lzIfKT;_}M3Nus#CkJx&cxZJ~> ztqndp`c^plN7}7p#MQXoNdm%bG9HJEt0>pWi|TDYNxW^}rrLUhc)yRg^%NZGNsfB@ zz2ASb`BWSwJ{jen+$ZA~o6o}}@!2xQ$+w)b`D$@{SKB&S+<~FCUhApX*g8erF$uOV z^3-2Ho9em8Qf!@#({KjP#92587vLgXf(5u7SBg97M`Qj6SX-wumtao@AqX-GB_%~q FMhcn7yLSKp delta 37698 zcmZU(Q*fYN7p>drjyg6wwrx8d+qP}Jv2EM7(XnmYwv+Si`fv8Gb2+Ql)mT+yJ#&l| zo`V^dgX!A{k`M?maq#j&IXXEQ>sv#)t!XDqT55`+_&$H3Y7{5#x%2(Qxati3B`ujn z-R^X7t|T`u=z{k8+;xwGC=Q0?#mRKcV|vuS)zq=SYwh}Z{AvvJbYEt}4G-VHz1>}Q z>bmMK)r9OPv33mie_aZGT;<@&s4hP~hs+O$96P&yJ`V%%c)Z=;k6$+*K0F^!uk$nd z&#kg+>+Ja-X)TSmh(gxbAEoR-;@2{ndJ)XCT|;8HpBM~ovPvC z@6jfOdf<9|{wpWszFCK=p^8nSLz5gzIgfaObQNo+GUe}7Ta=b6PB>P*h#5bv&?5iu ztZJGWf1Vm@rMeyG;PxP?AW;MhLBu;I3D?`fkRR$`Bu;Q-oQi&(A@zuT=(@Ka#GP;& zPfesKdDR4LM0XO%@}aN}FGw?|Nhgu^YxisJc>rx|*9Vt=-JZ3+!qWuIU)RJ{>B837qHvr@R6e>0;^qg-87 zx?CWZx<+Kk8$zEf6{;q6H+v8r^k#!Rx?se7K8fBYW+j`Dmp+1Sf|T&6aDfl6#Ao}E z2VkXkogBJ{pRa#QLgj@y1%=oTsWy#vUJyzxleUEENdj7mAoTaI8KiNke$9coEH+WS z-vm2%g#~}ynhKj`MHT^am8%)SG7nZBwNc()(9B-B9FoP$D*DF*k|&wVB~KkZa$aN* zV(kv}Dvr5{DufWP7+q{m8l=OUGX_|B1K{c8z!YrF!%&!uAJEkpN=-PfM1oPCf6rI| z%ZgxabJ7@rekN$-dV8+~HcUJxDR|U5Vi9;(nMw-+y``6Dx95L6a2n5qGNOk#-RY(x z5rE`Q;mn^Fo+NYOL-15?j8`JBdZchOL85(nfN)~8NljItASeWxMS~!}(L6os#hvzV z^piW*Su(#bBK#sK7{PBiHz*w8)H0D#n8LUspClrgw3uo1!ViT+GW^IZOX*mmfEx{s zi3Wp4i8f_41mp$i8bQlTb2n{`WVJw8mZj9;GW=oCh_Cd!mkJ+r(zzr3zFD%kZq ztWwtqniz2rCJ3LIQpxm4d;B7z8=554v7@U~Esc=Mb8woq$|ylpB>6tkeXyT~?zF^@ z1uyCT_s2e1=eG;5t*qkKD7VM)`-7bpz+EIuSJD=74OqB@bf)N6f*_}ahk;N4O#in7 zc&D$I>$JoNU4NpSJFrRLBpxmN_7@PNe=u)ppnIFE`)-Bh8aEki8(D8TdZEm7WgCe7 zr#6=;QB*Du@mce}e*5%#-%_&teS3G<-!a@$lEcdfc)5Sv9WJN9$CTjV{WteGe4Xwe z!*9c!e&hN0d;^b*=U+9~JzwkZIe4-)-xZ(Z9WB7;u`6Ha``yRs`tkMFet*5Z0=FNRr*4l&QOS;wFT3Nz!*sga zfLmuL@b%#K_*wVe<^JY#3E1lJWb=LH*t`Dmdfu1C`+ENz&jw^#j!RN&*aVM7mp39H zwR1tBum?}j$t|eG#n1{FqiI-Y-_A zu_dvfp`9ergt+*n4NE3FNydQ-wz&DH4ie}J@u_i=s(M24W6e*{P0OQZ+Sn;q8JFjf zhdK4Z?Ki33RW*K7%)|$6Ls#9x*M5_F-R3l)WuZs_49nIBKyFS(tf4m5%dPCr%7E0Vhy}94H2D9rI zSRrc{=u+c4OufG^iSF(e0WQv{U~*;vGMM>k_Yk(o>6_UFuC03bxj0%~Z2yoNNx>_i z<~)x)YLFrVZo|3NE`vF@^w8HWznVNE1^@I=D>f(0L(yt7I+dX4Y(R9caZh zNaWDg>t?k>XcpnTFf(N!*I%b| zLyA$A@2*|zou=1&1u!FyJ4m)7n!^o5G(gtXIWGl~cXzUHpK{gezE$`QmKsQ6ucqI@ z`_>}}>|Iacm}~3d`V{pVb`F3hvTV%E{0FcU@RHU}TA|-ptV*cUDc)q4xWYX<2W&JB zgd))^LtqLY=FJ^^#=|SX&4$oXBd+JESzrHB7|S&B7B>FN!CvFu2M;pGG)zK6|Ed6M z^N!c1Q!9JbW`G)R3nKqfNibSm+p6CxQQ#P44T%?R}S6+lo7&f=o5iSf^d3jzD z)|Ak?tzo0Wp2q~95RUqKIN)C$BZ9fnVyL0tPb%w)kGbQlNG1VdByhFb%%?1jLQ-O6 z=qncIH8(v!0$(I6G=cQh+CuUn%H&;TLkk~=7sx&C^=b3bZ+vPj)BYmSQ@;8uWJ8)TA25|8uhdq(o0Z#o9@~w51 zA{h_Yb)*a5GycG3-x} zzr#seIAe8*8?MV~B~v~NAaAA^wegVSlCp9wr*(qvBClC{#WPJ+sx$$+kt9GHat2f- z1dyTe3IM{Gy%g^f`hfaMl&~6ks|1qYnxO}ZHo7IX2Eb2W9snp+4i@`Tk1PJ!?v zuw>bnFKTgV(s_85w%QnDLt#NK#HNd&9+Jfh24IJc0zq4U(+Jf<1kLbw0=3kVkqNOQ zp-+i8&$3k{Wg=wzwURwv#I`b1rIVkG(VmxDRJhDUbKkIx;ZVxSS(dPIvImO8VDCXIbEjl5m+H98l+Ea zsX*#6XIz5SVI-?4t$xdD5(myes<4kkI;z?u>Wbe-;d**`E$eyxSS~LqmAHZb<*Yc* zgoe7KS<)|EsRZuUg5T~(OSDX=uPwYY5rjbb0c2SCac;2WS<+#zwaM%0)G4@l=E-O{ z)8b5_VL}pWAd*#O!Z|Af;#GRCh!2H&Jy2j%9y|~>ehRayS*a&2Yeo2L{7g0}w8`X6 zYK!51pd$3RL$Cl>AQ@R3!X?(WSb~jKcVT>B#%mO6Lbcw@@g0W=wMeqaW(-6!8_|fT z1}2{>{Kyf7C}me5dkwM-b=r^@hQua;<|&=E z|2%Q=d{N?_P4%)Ema>V#9in`~=~CimN~mcWgpEt!Qlp1Iue{sDaARI7OT}ceXV5g= zU}h~U5PwWR^zy6wH*of272!v^JxWZ*D6XC5w7?Ck}HNuBk+(>_Uy5HvK5Bo z$3LPgKd$dJdCDJlW0v^+>;f_I;4~JF7L<~7vzD*wHqWY)+|n6Q7)NEtV>)dRW!x5g zW_AQfsC%4G8gi*1?FfE%f@?ioi#vV`JLFz=H_op9=->zZSAm z74?awjlu0(qRq+rUmZ;Z75@cbFXplrKtS19omkaExG`9S+5MqfPQ57pKK79LF_RVA;R$gBEKVvbd#2PN zT}SA@hpW-!HZuE5RC8(m3cn(mWeJ`WOvLoMT&o%0HMZWDzw%_M34i|Tj5jf2^L?-a z>iE3VM#Q5``y*49ajcr#a5?y*TIM(!&#Pp@=U8n{xI~2wa5**FohnoT>$X%k+O;OV zOP^^Ak;*BnxK~4KS*1b)2qWG!cKdk$b1bmJY$g2Bp%Zt4WA;j09RBv6@xP}1tI}Ku zHO;lhb_Zb(HX&r~&%caN{hqMv&yh;AAb3ph1|^7AxIzftc%QWK&+|eUW|~R>ro@a( zNkl$)a|(0-!(C~MuObhQjySf<#@?libuc2*o!MhoX|*6NbD~h!<6lF-NZbe*#5HAv zl#^bS>0(!mvA0nb;~)%)f}%9rsGdML#H`LWsrzuDM(P1%!U|Us-*xC%de3rmym9%G z6pl`gQdrb-1O@g+7&sTg>*Se9LHv%k+nAsMVKg;R!BJZ@XgNv@t=m~ST(qgWcZr^( zv)aCme?BYMPg5~C0-mx}=CHdAZy6VHMfe6VIJkU^k6hK}V3Ql!2;0*NY}jM92_wjA zE3AlWRpN?n+A|c_ui^CYInXDQ+iigUofXsLYqJZ7E>tq-qIqY*rIL9%znH-!s#CLg zYYhPKbpBA*WGap(H$Xugqp(*iVYnfnd}wkY|#?`okbHn?g{JMX0Y(QjBJPvIym>nc?UiD;KRX*89NyrlGE&( z$Vt$Wbs1&1)gQC!Q9!q3lvpU}-Y}mG$nc?%>4}APdnF)O z7Z5O|NmyFrjr06}ag~BHOzS(dfRFE0~8D^j=6; zmG|W`_qA3M&=F@nfhQryCXs_Mo0{u%$Z5X6JG80vG;r`;OiYl`R;6ROs^okq zTZ177LNHKB!|{AmND9r_v#{iLJEUTi<%ya!#OwI`;!4{!k7`v%SRnb;fuX%(E!{?2 zJ&x)!&EK7wJ1rHxKLO0>9Jpl`S-VyGOtjI$r3*QfPaH}s*+>8%1tk`Mu9V^*nFpl6 z$R88MV}D}Cf=_aj{Mkxi&?K$%LX9iS4*P&JE* zq7?~3%Gx~pD8$36&n770ooV-T^M%E zfm_m-je@hQ4g1Ph_bU}s_SMkbcGAU~&BWa`-ACjJ&i^iQFdak?5Hd`ZJOmewl_Ytg zho-@Pqm(p`$))+N&iiuB#gtzRp`j;4eou3&(^;Z*9CByJxfT4#w|wuzwH=(Ye`=St zJ&m{g$;~}oGdwR*zH7I!jTZR*-m+=jGzDDI0+(Ay`dK-}w%d5!Hh3+^^XHdtEt9UN zANzLC4{Aj?aQDyv9iNZRkpRt0o)zD=?frWH=^^*Vc1^JUXVt4uk2YTj-L0KZ_V>y5 zr#H{U^8Tsw%Uxp2i^;ov+ehm5(Y!T6#U%I@al?8J{QbLeOHNJ4{CQ_P`eg&=mF2sY z%KhohdK-$r%QW3P;JWOhD_g(mx)g(rzd6IsLdH)dPXGnLs+0YcgPxfxkHm?}-)7$B zOVletXP%Y(8DH(mu=um=YTiiRkGYe)m+2Bi76HsH4MzUi+{id^)+Nyb|IAe7?^@c` ze7F=B^7~}Af|!c)72)iZi>M8yuntkmNto2L0yBr3U5;peY}|1l>u-l+bwQ_&0m-5~ zJxU866KF0VDXcwZM?(S!HS7#4J5P)hOqgDHaJM3l5v8E)hOvO8Hg5Yi++ zi{_?}7h~J9!7$#ynFX$(@GDx1T;qkuw$`jwX#Tn!HL)9eY z{oHnb-bUv^qqmvFE4Dah9}p@$H3=@n_0zi#M5r~0a*E>%?(x>TGw3ZUHFIDhO(Q7W zsr5!%lItVBF!!2Xdv{>2#4F(^A0QG~Ho9xYB4X?sgtf>uE@(sWtIgkek z&-PmJ$3a2b{ZA5(-4v;jRc_Q+T4fPT8-A6mHCTH-o+y}oD3P=LRED=7qvfX{N>R8% zEEKGg9+W-7z!#$lBn}sbf^?A(-x~0@B1d-5r(I+);8FeP{Gv<#3fRIU?AaoCk2!DR zfxgk`4Jhb=jV^gz4mfh?cB1IaTO2|8-7{Fm(M+7x`Q)9+5y#lLP}w2m>?)?tK4jKH z->%+OG*RP*I zI>#iwp5Mpe8UGW^B^N{qu-46u&_pC!BCJv@R4KM_cA7yHu=)im{@agK|98Z1VxxEQ z5z!R_@hZsZNKp@`@TR_%w0)%#L(6sZ;pXgY^GlaA^OECkySOy$IVi`Y2|mYo*KLQM zckiC}_vH6kB{z1Fd8di5F0P!(ttTc$FumD|FisEOPEjT9;z^;PgN{{#;WY?uuVRBT(2QDJ-(A*(61oh&lH>~*1QAz%TEGDxC;u~h2p zhcbVXVpC#Ry*zGtM06qgOr?Sbd5TH}<(-m3lUBLHU)}Q9k%&ZvFpcs|_vFMBs_}B6 z7ztzbBn1_v%47;Cz!)k`gD6Q-vIwe4P=i>0d<=!MO;}lrl0vm?yl{~*DP3xm*dfjT zMB(p!0Me@hraN7{B=jO5&(z6y0y*^Efy7@wYF$F~h-I>{!BNezF&3tTTdI52!5T?v z>A#V(Xc3D0^O8jpge9Td6!Pejl3~X6nf> zMKqo!N``KUV%q5MCM6PO6$$7)rZJSW7wvcjbFa|b!WnBxb#r$x0qmZJO8#;s1RZoa zeJ=}!roiX|q*CXHeJ9FC=B5x{anu`~!tvjSHF6Rbv1i57jcLs@Dp)5bYohC?XlnvI z!X%34=46-oz~+L$8A6hz=ZT7i`V7bJ0;L7EbCQmmtuHcPWFLA<1q?n1w86llW)$Rm?AFk4V3EvR{UG2|B$PUlk8rfeg`DKb9Sg=aU zkNhp<;5cFsK83blBSM~W+U%uRa^y;sK?eLR?CkXPzz7lvE?!GRj3`wrB`$hMaNrC6 z*le@9l}D(hnb|_vKfMMw=M(&?s6nHdQ))E&fm01^O?9>vD{E_Vh{J(=uT0y9_DGpf zOGnR6!diidRsE*jN3%`8IV-qCoGQihWiHYx=pPg z4n_^xNcQkSrM>JUfhF~EW;ql{bs*>L0%TA1o$2;hO55-(L?2wqJQ%JS3x}n)Vol!D zibu+C>VIaB0=gEPqBU{n!w6i9A$jGqZ-wtAe@!UxkOmDb0prQ|KNxcvm| zVY+`L>~uYHUUer1vzq@l+I=pe5qbacmNu#TYeX)Ee+!MPUTb|zp)jeVPXvdJlI!g7 z#KU@$9J;^thcG@&`z6gb`(<2$BYE(M0o~sB$SC&JRf9Ia&|A|ll8=r%Za?l78b>!w znwEbQEc>f~qD?y%Og|;Tq4j9=ZIm5a3VvPvUYmt=wC{fq=4*MdwlU|6am)qT?d~z- zI;1nrp1c_S2`*N=OaCVfZ29jAq_Fr8Jb?FUc2Ku@5tf1sPTEWd0$ij!rcYe$H!z0? z4=t?r1AA~)mQl>OkLg;gCyLPb^>$C_T!rU-z+)Ddomt~lu{$4k?U5ki)}P*1^36dC z6zS>nCdRhr>yv4ggq`YExm(Le_^U*kI3@n>GcYV|&MtU3>oaDoHf%dMnV@%>+>ak& zI_%zvSEm1Y-}Bb+g+cg=TskoR7?_B7We2ICgo3_rqjm=Af^CYSWm}@T&)@o|RVwOsP)_^=H z!0E!hq(s%pUzH2ZDc?%!T3ZosHZq1Oe5li=Uz>NX{2VEBjUg)jKB8!RSbIL+!s7A4 zOpP!c>_O~(8;B1HZz_gC1<1ohE3&Nph-i5AY_?yN33XCq>Ckw4??XpW?A7c%f zquf%I5M?OfAgctxB|3W)l%q99_JY7YmA^{uyAIk?)ll%kh;FfM13C7o7ozgNl+G-j z&p0D`-CB8j=Ct6}cjVB0++d1g2*4yD76x+P=jAbU{B-}ps)a)%(ej14wIF_v`Og0! zZN|00I)in$2x|gKbzc8|qgG| zz4IR0SNx1bxa0xsfDA~N!yU8CwM6SU)PQJn3qKQ=g&9?65V0)J8?&nn&<8JKe&{!C z5Bi8kIXT_3Qf1YV1_J1VWh0=1yXH0P4fSCJd;1I7M{kC(^Y=}`1=L7jB4MXhiwV(5 zm_kGX`-OPO`tErxS5_C!mm%4#L5aPpVLS{S-SBP@`^BrpIh2 zhkfjN_~-%1a~~`oX_PJO5j2-KAIh#5OrNk(CBk$cDLIHFg4e%w zdQ(VIIU#SqfDpk~-(LB%I2M6{Jl@CNUbDPKF< z!~u(IbQhFUFqM5geQL)71cm?aSHvJxSy7)wW4hD~D6EWGlMX|d%E4qa1P7&oUGooO zxx4fSr&H8Ta|gRNzdu@4!i}$O$rro*H5oj_ZK4=T0J*Q@+g1FoL{5Iwg!lf)2uJI{ zM_~EN@TeDolEWX0z>gFcy)}gT(UZ;{{T@F}!^vx=&|FT~{x4)Q2-(%Q453z`rRJuC z$djRotymU+3zT3-nL~Kv#xY;>1rUxWlNnvssH8^IRcxV?-{o4alXhV>4z;_fyvo{% z(4H7`04><+B(d-$(M2`cMSMP<6&EwL{lK5$G$1+Rv7kxV$#Dlve>u`j1cxh2N@4}( zQ=58?uT8+M#Wih9+{9nvjqC5Vv$w}}WW4Jy-Z{_BSB=n|-8~p>QF@_F)X3anpaN~K9`9_QhK??}APqJAiI6GX=RWJYQfOdtJ8E&Qk&9~tpTSv? zsj!*&){A)fli)8>O7vUu3$dLFtk~uV?%zMbzXwF(5EA#cx3+T3xV|>c_6vtb)mkNC zN|GrWCZ2-(XF&TQrXj|~rp5Z#H|l;9M|a|?^W3%nZvLXz2>V*p)-(!YuvW~eQp@t0 zX07-f24B){tbx#AY!N8DFsa@5oBYA zWzMLF+?)&o_s92a5T6$n!#6-d#SmmR6dV8};q1f62Ztq8kv{x9LbT-77?oO-`YhLc zy!yP}@|@zio=Yqu5t(n)p1I&GRNNh{Pm2jc3baFB$4X5z3sBB9zXKibADT zL`8(liN^)Ah^aO=m#br2{}x@v(=ad^E189A?V;y~Ck_Udz6doE%OPTsD3VAfy9<{q z)d5b<(h?F5zr(^@<;jUyIjdrcMal?CKMb)xRG?21fDApB9RDrK4Nu=5KVCg)a2>)h0Thxr;|>op96b15uy>M{(8O8lCtADm5D04(B6S9B22<~ zq4~l9mW-wCRS>ob)vrzcXaqYQ6Wt@TplW)YIsOukbQw_}_2bg{GY9?vhr4^2O80=7 z*c@HzU_tFVnV8}~ZT8@7(#{&vS;%V{uKkOrN<0ObPD*+%Lkgqg_~ z@I>G03C3}$o*)0i{znUx!8}cygjI;w*bh`RRFsJC1hk6RPE$LOUjb=uCitRw2Aa>^ zF#PFS+R+$|xfL?*b%iOdzc?U`mU(d*3768;bAQo{QOu#%AJRL$-`!WIz^4ARiny2r z!#=-RRoR##0~RTFwtCB8SbtMl=|NZlVdzmxIK4H6Fd{o1wxo4 ztHxe@Z*zT&1G+NIspuFdPXbzurO=7*rK@&>rP-z8#Hckst6q7SM(wON<^bhBWv^!Z2Ay3%Gx!&yUypQ^I)q> zEmTt=d?BoYp7j-j`4o_$b)AW&mLZT`mz=Uo1nh-;*C%1-f4u&+^19f%8T38K2R{j< zOP^7SA{X>DLdB**$ATW(c{t7a2bguNpwMU(@oW)h%Q!;I@(!wyzTGK)+}8Tu58`%m zHc_l{F)7BFfbqlN?~yUHetc#Wi+28$^ytw4qaXLc<)dJBUccx$70_K>cBjk^SlI;Ad&mraQQ|Cj+w4I&{UaT1#>Ky49?Hx?HEUhQp<*v$F7oR_O*t6bd~KK&{i49O1;ro0|5Wd2+l_ z=UAusd9e^WJ9uuplZNAFhSOT0?b~rfCsksf`|B%Abjjzi-726YF{pH|^r*DH|J9u> zAoKa!8;^T7Kg^o0PS5E?m%8(zy>=Igum_mn3!QXn z!!+NUNt}iP;NPzqtXDccL-%X+(9hi;?binxZ`MKu;5MT*4-RNA+(L|7aBEXT{j^cO(rLk1xGjKz1Z6--k>hIxUTg{1PgA$eY;ZAWqoroSoAI(O^ZD>HVa^rd6 z`&;a#U3i=GbV>y}YU;nh;uGOqYWr+%R%9?#6kwzOej#Al%q4VE9e$wE6IL-Ep)8pLJ*qUJlz1VzDgz( zi6^A3$~}^2X%P|{7mr%BoIa5>vV*1b+TWs{g^4m*g*Nzks>?ss#WA^2 zW&@M(UzVPkaDVXbjtdC`{)1q{v;6*Hbee11`kMFo5MnVRh6ez0-23I`!abD`OKk-n zt+PDDYjEjrWcgUQna;LAe>8V#D6e;yhu6q~iItt;b%R@!?t|Q(c;bE5cOJovDf~n2 zJg^`0=IuG?LU8CHS)~CBm}4U-(R7f+9i@6N`r&=<{1~_~aaxc^kpQ3h@l?X?j_R?hEKeiHc5VAHujn;n)Rj~YlW8>oK zHPPMK`z@J(;WFyIr%pi*_Ew~Gn>bAsk;B0g|B~7E+SdHe{e^&|dhzrq0YZ0ZK!Kpg z&$9x5ksC$$1v)GMLb?N*83<{|`NI!H?H{rm%u_#&c8U!kEWKd)Y_H~*v3Blp2?LjL zN8)s;0SOD4#Fy%a5Pyg?2K{(8Jsxq^OE^IE{9pminlX1v{03tG+P2Z@izeEGmwENZ z!v3y|xV*tKO^iOrG%7`zCSXpBt$TBGP-_gwCWA_v+*wNC9}J!#v_a1$MqQZqN0MZw zrBq*!919VcIh=TjQN8Y{iKz_wJBfZ&H*mP3X26_{A;Lh*h(3AGWCleuY4}8UkI&If zrWJbV5abjmmUfGsuzGtqAhAa(76*>nnZL6gA;(jCvPk>!FHkiNhq*r7mdU-5c)S$w zC36k!yUH~Pszf*hoT_Y|`GQI9HG5-FUo!mhf8tJQGKiNx{L z!%;AY4Wnzd?KKVE?G3GKGxO7vT4*P32f^NG5$A#z!si+mv$5hiip2f;f(Ch?cyPu) zHCC~jN4sv+(fQ=*(Wd)n;BDqp+F62s2M!t2GJWd}=Vz6SVd+vAv6JQ^t`00*;Zv!& zHqC)!St_^u_MIb~uD4@Jk{n|PwSOC|HiCa#E(EQO#nCwgg&pD>_gzDMxxhBs&o_Hx zNrhW4$r5&@^NqWpfiZ`bIDXM?PorOSQGts#U@6 zotD2Zpjvb|vh}R#Qk+dyIhYgnO+rlG%)s?*>?vY~$1Pphm9pKqawO=CXq-?SJf!{& zwAb9dG0@Q~0p-~0f4CysbcgJ99s;bNF5QCA7oyw2HY>2#GaJ?@AHCCk^nNtaUlE5j z-ItPbG}FsQfo_&!vojq3ju>aO=WqJ{az`xE;(FaQLrzbV>y{QVXvKem^~ALgqz>JV z`?ZI{A$(-++u7hT6!$!U%lK-#l5`ecQ>qaB!vG?zzE9N4xSO35fy634g2 zIMwg?=Rn)?UdF}j0(8gl5h~k}B?{SO^%{Mvvl{vTI3IG#cn1hO2y+T(CkPdwGTR=3 z4H6GX^L$N3)4FQjS!3OtYux*G&bB)v_lt|kr918W z`u&GcsQb3(l;f29l;ag_1}fT+LBr}U^fFy|m^z<%Z@sZ(@ib&NBNuAMRr?!CK-zEW zBXw3LdxZNxA(*J7t)Ct%+rhAaNBW?_b2KDx_m|p@{fFJ%NIE|*sY|pGtZ4%Tytfna zcUVcmP-G6{=U_We$Ud^;J`$3mx52`LdV@~t9~L4`50!<^#(@h!dwqYdAQ{Xy z1uaGLOFT*=sm@!~pqlt>b@gvsTN<5o1}Rcl@lesB)a1z4Ye!f@fpCXILjIQ2#NUZ2 zNvTkc#JFtiCT?0928(~!WxsmRj0%mR9FZFel8uRpO-hijMPUqs8boW|FT|e|sX?g1 zS<+ej9_Mnm>yBV`7!M|aUVEW=T;nC+njJd&DXn?fJ4 zEEz0+pF1xpEMZq3R$?qcElDh8R4P|4RD?BRjMrkQ~_#MDj4UEE_e97 z_&gzd55ZM1Tx>6Cy(O(r1P#Gn^q}%Ja%g8yHA+IJFqa+&hx+q)MYxN>Av{m#!%wiT z(JS=%mIvXXqlhoJU7gfA+cn!bB*)M%Ya>N*S+bQLdnIHsyiI#$*s`wBGP5KPlci^- z$-MWQW~(PTLryV(Hhna%tEE?eppq!|)Sp31?-LO`CJ86OK)5i`uPE=fk5#5{L&WdF z&Oxt{bM0@H#vk{3XxVWr;L|VWkh&#}XViMXB6{`OBA;%uDr9V~Kq&WUu>!J%_00%j zFTe2Rz;fX8x%hV2r%IPE!GuT6q8MH}<-X2M>@G4-i@eZqmeqmOWZxm_@4N4lvg zEv;Z+o0$QyPg=+oe&h^iO>8yn%S|}5OC@lCgPcZaP{<@)&}boGix^T? zKwaoDDi@7;l6Y|VmDlg*%@D?rmC7(v2bV)5S`x*T)CV(i11X(QQ+{vASu~|!(nO<0 z)SNl~gxWT%;#gZkQO7oWx&kXNV7BIhJx%{WVkmxVxqT$zWGOBC+0rSs^P^{VOWbOU z_&E=F#-4rWed@+tSnJm}z}WeWz1yVz6GXNb!0y<9Ec?#n0VOT!87J;5yKCxZ%=4v* zqx$sF{D=3)LUwuNU*?Mi8}VWNM9QcX9)|0vEZf2w*) zchc9D;WTbI&gJ1?dt zoG@y;v5p_6mT1MIrYNI*FM6mlSBd%B6*@Ioa^D$C8NfMJCjOPh7Li5oWDMa|Zw)_U6V&nSe_QnN?alx#47 z$#^z(@h8}cBr_No)qw55ucYdV$62{cvnG}$BD8I-)3bUdwO`er2b;!>rXcRF*`jI9 z2-im`q5P_ebP1%D?!QWvni@JVSCbo>o3s)mrBvH6&eK++$ zEB#Wduzno)Fm&K!IswJrz{irvSn;yOAKYotHU(NByV)uc z5R|3WiA&P+E<5&P2&?QPNk?=4uDRCMd>sx%@efJlrvrqH-$MU?Hu7C2twtuU1>_h793paKnWKxPG=B)+dd2Ed3L; zm~C6!#{asNmHc~|3@O21vRE21sJ?zn#7~oP*d-|BDE<6uV$qVJdR$6UqyUeWrAnlH zrbi}@k0*@g&k0}hHd~K;l7VdA9Vb?fsR_Fqfehg`_p^3JkNP{Hsr)yO&*k{`FXwLi zeT#{~n0-Vys4i<9V7lD(j7og0CzLb~(`;yU;#nv6R!uFsrMi!*G&I9aT=_!1^@sCU zPj=hcoiAc!qW16h3DujI(VzeHb7*@g)I*MMgnn1#IsEh}`1c#tq+?pR+#-EdEU4he zhW_^#97LCW>T>M<_+RwHgF*;lKX6@qeEjc-<=D#Zeo<(zKR}~QKNry!UQT=t2}-gt zD>U~GBiA3%0uUH}%)HvbV57%#DqXtdxI-tUu%ZEjo-bT1Lda_-lNZdCm`}ue2$^kA z(i~NNHYAHYEeX#$eDT?!vo9B`1Ia{A)KURsIaSKk9cbJpd0WsivJn4&mQd2 z*Ka!^q$i1xUI5FeVn%Z72I5z~o(RLOw^ma$`~z(h-J>e|&XNF2l{NxVq&m1z6UT{bXuqUJog_*H%@P#|aUe7^tAI#KU4CAf&x%8F*_$2~9&%$9@d$8zTT$ZJr?_4guaoezOs)T0EI6y}W{cQccF-?Q z|FlW4BpEQ2nF@jH8CKeB=>2mqETU7R+$9Pqa_UPUDTK9PI{nr(p#S|YEG)&Vv_YTv z{Ot1>1$ZHn*qi}0+($TN4j2+!n57P(p6R?26lHK+lpEQmB_}X0K(gXNKEwW6Q zqK9OsvVJTu&C}-3GBb0`dd>Y7&GLGOFX<-jhUY|*K2T0p!8kxC+eJtJUD4(r_z24U z0BfW7Z2b`%P~0>8PDV%PIE&T@LX!W7`H2k}G&~6j5Y}em`Qd@^4BxVB2m*^O?Dpm8 z2FQutQAOM!?7=(}%pf4gi?3{1@LPARLcA~8y(Q=W%@Hwga|9@P) zb95$A)3+NZ6WjL0wr$(C?K_;b3uBtku=sRl9fn z>Z)uBRqEGVDK7Aeu}xR0Yb)lgCt-5jLGkz{@;-n_2q6pGm`|*Myp1|eHzf0zUCm7~ zq#qMbZm66dYy!+HTsuj-ol*dz8bktWkA$vVV_EE1ycLU&0|w~!5ru)EOt*kcK=Ajf zps?gzc(-HwO3l0*Zj@UXg@_4np;}aeuT(?U)HZ8Hg5$D|I(Vz>{ZZ5%4C>tG{iy%W z6BVG{N^Cez(|Q0!#6HcSn3fduOS857OSd(7y@XcUUgOtz1uR%6&HUu1(jMvgo0NDL)&_$0uGsUDHqsh`l=b!{K~h1d#qzX@eM%cXZO zoFVHS)Kly=*QCP{!_2mo8zJbptXFP(O<^f9B5r*a9<*+(7$2>f*eXZ^$b%XJ5Q-Wu zI8oF3?l$3b>b=G}n)5H}7}|L%0Vmq29i>WlYhgbvuv~io-|s$WyL0tdxaTL8U0VR^ zJP^uHaMd$%&??i?+b8+ggA1yYXvgt1yMWKWBD4;JwmlS;5V!e~PD-#z!f8~H%)c^` z#zZ!I3}y+31zqd(?P8LqTxxpB$}0eD{vIGkz=I7hSChDLUbrAd^hXd&JbWNJ;W$MB zE(Nf_561gLg&Hti3(*(H*R*r6b_g)?oHuG(&gk)-V-v`fAhg!$wZR3Bi}po>5G+`IShaJQaiH{f3g%(iWc6?I#VSp7Q@nH4d^ z`j2;ZAle;Qx#DE^jO!(k!+=$pvaOI$?K?|O6e^hZhm09B2}c>TN*r&in)UemZOy>u&94Dk{{1BWI>4v;5lksRpb@Z?Ko-6=9Hyqh@30I&D;A0#5j71G%xo% zF$2%8^!2>HB+eB;X4ULy+nLU45`a_ulWonH5xsB5$JTu{jR0tu_h)*eDTCy?A(rVu zV)j8%>Zl50p`1KJ!r@Pm%{!$b%#y=W!XQF82+&7KmJyo@vS_gK&SFg&2Z&k?;WF`zPkI~>-he-jh-Ps^Fv}PDgd6-_$ym~0F%BI5 zXDOBvwsHrEH&Tqp$Q9nh>+?@uQi>oJ@i4`GK zkO8M9LbdzT;my9)&`XKwS5V0VDgMP@RJ}iunoD;nncYyauO*OHg9y7Kc^qA?1f=CE zw{@46ZQZf)p>CjZ=V9Gwwvw6`AB>Abwy7^l`rjVpZKT3TV`UhITRzs_bLAl0^;65? z6A6!dpzDeB44PAmB(4t3j6&+R)ZXH*{Cxuw+ zg((6MP%X@dLSOp5!Zu1>KPGUe;#d}^)NVFjWuXnC$tlbl(orMyU5M^e6tD5toC#~s zB2Pr9X34ryf1bb&55&}NaDZauZ!C)rj%`ubG&a556>5mcUwc_32C4j}pFgi?GyoVl z+gptABr$|Ia}mbp`#(*NM?Xk&HTMM|VK{Gf>?bte?rcJ{b>REjv!~JcisJc9&&m8c zm^aZ3B8E(n=D}h1DC29_{N>?ZkCZ2NcaNX4!E8k9TYGL5G4&^;X~q2a8h#gdI|i%l zm8h?y+si5eR>BP$;ypQQ3>9Ui9<^#ZI+lO#)R`YaIAk;wp>Zo94R7?vRD>j_AjvcT%op2|Ru-ah8mdA#}h`Yu66>a$EWU9FiXhj{C)!6nk zI=+oa{-d)da;Th{)|m~5t(sm}+7{Vrd8)I;)|nx38t%qeeeRge(e$R1)AHuj?(f4KpxE z#Xp-i6F2_erMQF@?#Pk2ITtZ$+<_Gz1pJWJDO%ime#jm=)E81vh)U;-N5}aCrm7hN z2_Y%My`Nw=u^+XjJ=I@eU;hCzYS;SCJ*z1Axy5;3tsqI&mT}Ih8zoHLojvF5>;%5B zvaBU6WJ2LK3YTS_BLTqFc;Z3PXa> z7`ygF8Qw#vm!f%uaby;mbO1aAkP7j&-!vRU>@!urPe{=D_Je!{%WrEH^od8k-DD2DN_T#uXb3G=lugz9PFT3GmQZn#+o1@V zx91g34{xAv`Q6-i>y@JAt^WA^1sZI>fK9yS;v?mh;1slM?DI;;4sPrQ5)&_!UA15| zmD9~nv0u#x0N?5!)9Vj{oth%EQYR9FY0cy6`|Gt=b6Q+SG%>$2v(Ro9?Jy=z*^Jan z&Cc^G+iHltNikHvqwoZI@irxgnWF+_!J-Miru2W!nLJIk))UX{UfoqgH4VQwVCbgf zcqKjCzz;n!sOA!iQ*FtU--r&QkL`6@aM==|O)USY0JMJuDY-Sdgj(N1m%Krw??zu` z396vv7gD`i+}hk-G|HLC2I;DZ(^XQGX-V@Ne2(%X@AMoqZ24&UeL=R_h{%}BeUogr zQTu0C4v0LeW=YSaji_=KomuB!*sN8r^_87a=<8`aB~}Yjw^zNMMYjlw7_U@ITZXL4 z#&yAKgV7jF3)M7CoA4mNA~q?yj`Q0sID8>QUkUL@gpZSi_6CRc2Y60PrexWvK56x` z$nz;`*8EhmMB&Efp|XRJZ7FWS`uQU9#!N~D4ft?0KQ+-xZubv+61)5CxB+<8<4kXM zLl49dqJK@mbmPQ#6tRDe=W0i8pj~yT9@sRey()MVF>!s&ST_A6Mwj(O!7};WhP+;j zTKF<#AV+GizXr4E{bS5g=(D{tQ72|9@X1aXouBD@u#|owwMeBv<;igx% z2Qba;>i!Q~gaJbo?Il}zblZx<{DJGH2tQV8jSn`+VMVmU*N3qw2|n1f&= zsrY$PT5qB4TzF?Ca`^;cEGrx@^NIFu1-!mOIaW?|l=prVb{@3sZ&={UN$}$E3}%Xd z%|!PcC(ex+r!m8M6*l2fkYAuvNq4H8w|aF8K8Wsp4buYy-a_0VHU;~JdNtA!AfnOWhA*A-me_c+7R3`j*W zGP`(1pH6l~%Dg^AyRuwa>l0r_3m`J7T#QL)q8JV7pqv|vL)~$xD0&ZLNMzFQG#9^) z!?SY1g5{+QWT{ZJ9!LDAwcK72(PyU9#$6D7r>^&*GyCK1E|<5mwo}aCpmbeX=lbkI zZc`9CUIrIWQF1d(b9>@90|~(eqj2SR;L+P^PY-@`CUpj5N9yz}RAod!9$@3w-)vdw zgI4ih3jL`G)#_#ByT>o8gcqo@`OfLPF6K!Zh~kOaPj`YH(SWp%p3IvUV$L zXapSxkHTn!WEPgxASK6SRGkV0N%}l@zm5r;3#j&aUI%gVESHS6L9&Q+i0+T_}@T#N`$Ucu|zr+|y*rM0t~$G2HHKV5k%8pPb&@(>!`TzW;V5LGp4HL-L; zREsI9r?A%5SP0CSR4%JKGxyG=rxIkGuO1!9%sriAiEo#eel@ zcBmrSKRKH5Zy+PUt--r*kGJPl#pGQwrY4oPb$?nM&ctcuMy<M) zK_sggSM(pRSb4`@k(NNf&w0m5UbbHwo}F194g+V=`*@4lAE~)uaJt=ax47iPfy0}U zw~Re`s)7&1OWP3+ZpU=LX^!-qdG$i-JkJ=_^Q}ewP70^XbTE)&nym0i4vaMargY2W zOnBN+&GYXcs3%Q_JOKWT5cAC0^$s6fb$rc5BG+)=g>vypk}XOVW}Nipq=@y@wR0KD zsF{la?Y#POeo<}&F+4>0dcy37cWB+8ZZEgKA*T!Fxt&w;i4Y=1E}m@cJCN>o797BW zKDTcke}EirK0L2Jw#{OwES^r98AR!%f$u9RbH3W1^O({eKmaC;HDg3_-ac7as#Fh_ zca;ZrU787EG>SWLoEe_+0?%BA4ij87j>kh~Yh4@f zRQ3@gfpCbTu3dn9&tYYhhR1L;D>Zc{jMMMh-_*>R-!VI2YDw)LLVFG4{4FO^oGo&sTv$3K;Ft_LAn7l?+6o7;Q{@)2Xxz(1$iZm_Le&OV8k-9=$ z|CXgC6JOzgB)fyx59#t#!2zJ*GTC(7B$}V7Q`7qEy<+|3VE8iy>g>oUtyG zARTp=#yE1Xt;f#lIw;#yMv7Jc#24azj(z0>c<#l2(SU*CU??m1=4O<;xM$aXSZ|Se ziTzQD7w&kZ)K|Xa)wj$YC#50z?N2iLu6U0}!s^1F&gET0Qmu}jSTd7S)kJQg z3I@;sUK|<;Xoc8stWj*8R<vFkp>8xZZLcBlpkNWBCmRnB#FQGF>EA6&@2+ zHX`8tauEF7NzxyX{ZIb?ZsS{n&&u^*00$A?)rHsc5dCoM2_kgGceeiDvpE?K7`d6XKmQ+)W z60V=|fHj9_=lbX^nBuZ08>-yzIyZL)++%A8ov%M2Hf1DZuIX_vNki1e^R*xgRkR6o z@UB^xKyFZquPtMJhJW*s*UJUXLe|)h#W`j%-UwTE6LD-8*H6U-MljplRwxK&>51n&Rp4M1H>N zSp3;h8uAYt#ZD!q!XwIJC^WDeF$2D6mAB)Bgm}n$(l#$#Veoyz@!Q}X<{3^>7sCI? z)G=G2oM7Bc>BC!~^gJZUK?w;$t(0)mJU!%FYn=;kcfejxsx4_A+45vWX=1z}F)?gp zNO3g?cRvS3xWgr_wCioB!E8^Sbh>R&B0$IeOALLNrrr4HYNtTJfi;S0hrg>vM4J31`FL_?!S%F<_>|BcWC+`H@Y|Oa;9h1%m!QBGp&}5Eh#Wb(1%%#*NgghU zG_M`mkiFm(OGJDFCn9 zYv68w;Orqr#2M#zl~PQfT3O#8Muy|a^M!PNeM-y~JFg4oNU_GMdYR@ZMmm3smvJ}g zj&?f!XsZ1m=Ux=2*)L1`54mCL-48E58dEdb&+6yyGzLO{pi5d>nuwk&>D|d?VZa@3NBHpM+_!Gq*LLnU zCFJKd{gu>~>yCf3KYE^^0-)}g#ZIfA zZ`=|!qs0@`8Q!LR8Bu|6GH#_3QNin4?NnU|Jzd8++9xMeb|5lGx$EailXf|Dd|nt_ z#F%lO@yhu*f{IlnlI`d+B8gMT#>q0-S^A;_$G^oZ8rlV$^AcRL9gXt}DDqWfhDsh= z2dcNinfc?$C@@K4XbkHkLtXjmv%NRhZuRIB4O72Hn za-~}e_q;Fw&3Nub6YZ3EX`~gZ#(p+(dyH^>XbWVmV~OuEQxyEI5IuDNugqy$?X)x%s0Q%BSM^f(YCY-Hl)frz3zqS9y+d0nhb&f6%ei_;D8Kf5WZsK ziMY-)knchV3XGuG1^kfP)4Et(95NRYC2;p9mr@V!AEnFKhz20wG7%eh{hi>4oz57J zRz?r@4>xs2FL^n|EeUgw?SV}iaNrfY~#gc#5B(IER7Zw#X^2*ZP7NW3djENi~dk*W-e zFLa&>Icah9(}BjFg`p%?jWl{ZhFm2Rl*7(;dVF;D6ai|M6p*TVaPxTyUak2wYx%oR3N_rv)0;3_%0fWrt|tB6M80Mt1{QXq=IX^K3twNMG7 z#U=elYL(nRJqqPZGqq+4&JIeNdZHF9x(UlyVF_a=w&bP6EK;#nl6SDkS~p)-*+T0V zVXaz3Y)J*IxQ{GH>8>I86f*_uFj6$omD5Exi6kd1fN;486RI{`(_bLOx*)-z+-wOp zR==QJbG04+9{;^MX>E>EqUBmdLoZUnuCL;2(J;@_59*jN%o_ZQ2`S;gHm1|zfX&<` zccOQ9T(I&6q79CG#~JdAj(H3}HiY`I^^!re4QcLc*n^0*s}2PAK;)gJf(pmO1>r=P zGE*z(O_Vz+97Sb>hdV@ zC$*8Sx3O)whEi5u-Gd93BbrEV@%QNgzUrN(Yf9 zgz`EN|ApA*DSNhh%up!(yCCN(e`HVDxXfs;iLCAL|XWJ1${($*N;S#N2ki5Y4nR^VL|d;L-z z69BTN3DO$IowD(w$VHzskc~^1gY)s!O3kkDaz`P0`Uk^F}Hh z?CB}&?CfmVv0k?9h+)aL{SL@CxtT}ptYq;37&0c*GXyBUg~456k6Ow8rTtos%4kHQ z-9zy8>UNhz2xy3@%Te#FO95V~4~JcUN5B_(SQ&_WM=1vR^5biHW{N-F=Ut6NNFgRQ zsT%P317&&WO>J;W>^1&D^n$du29>;`Q}VEBN{dl%bv<{C&b-~LnYZcIA29lXkpO1a*b>tnuuc)H7cGw{Pyk=?*tkNMiYTYCBV5g_(@ zav{??X4g*`YR8kFQyn7Hvw9O8r)sN&T}a6O*6)kTCc|K*hM^e16IqQdc2|}`U^d(;^Hcre&x5GKUIWc?ShZNo=|SHNX;VV;2u@(Hd|u$(m6qWp zu7Ye~XJ<)KeMhReWBnm1(w)HMAFuK z`=OXQqA63GPU%QzNrMpKuq=@x8c;)G1gWkf{zO8C!LEBMkA>+?d)`8ntQsg)z@K%v zI&YOK+fa;d<2~&w$O2S}BDHbLMfy|11 zj?dp|D+XB-xaIIW84)+^sGO4w=Vk%ICatTS8(ek6|vftVF-gLk4@9Rm~hP}b zDra-^%W2eBwyK8QFY|zV?IyhKdVTqlyNbt)uArg)R?o~W3*C#B$WGn$y!FLb*nbQZYptR2{;LLh=0O3Fz2}A;@(ku_-O&i zlilUP00QtuQ>^9BiGFx!x@mojlkk9{Nkz>Rx~Ww4UH!BVH*rPGw0Al2FMLL44DK%? z_1UG#UhEbVKyOmJv*)wfRxU)0(Ww=R|geK`X=Nk&54Dgp2HK zmrw_YWJw2<0z0Mt$Q*BSCvJDX$cwyv@jvds+x-){_}6Mc$%`lL==}Uc*nE#k)VCBf z4-Z=fs~fa&7bWO_o%P9bwVcqXM-$BC$0g1>AiWuSi17+GguOEo668bRJqSr| z*6}ph{}mw0Ex%1=G<77O2lct#0)j%HrL7*e16HDAS%QSC>6VtwS#08z4eR%~g*p&K zlgV@N-r4-7x9pvb8D{XKKHsy2b7gAk70iSgfdc5@y7A)(}WPg#mMIMBcly`ii`iRfb=0E(^02JxR?vXFC7V^Hw)!*ftI z*!Oq+HSV?Qwqx^j(F@Rgz{XVCSqI9Q`a1gKdQHuw+GFR#1KlGBnen`h+!)#8 z6&so*-$@;C+us_$ebgP>biWgP2Tok`Y!bRzYdYTrw(6HmtB+?}HL)#(X6|SBBIVO7 z*xHY*k_&k~t=X|@G+-6etmj(Qkr`CUb8x4*YM?FYKIt^YTJtuAfF3r(0&Ye9m7 zjS~->fJt=}_fUnBF34yI_TY^H{sG}V&LXpI+GBu`lRFXF-NQUJABouBiOV<4Zw*5S z>DWGKaP|V|;iYWCj6Oh!LzvR4(>cM2!`SzA(2qPi5Y(!K8rr_R?xA{oRG9U0DE&VvzYoG0J|B4=bC%Y^ecl_SP-J05zc!EbqD5o#5(g`YUyE zhl#Zmge@r=&^ts?QFS$Vr)cMYMTWnaT$=qS6@uQLKEOds(7*`}4mk`?f(gV!lnr`h z?|=*IdY5Y&&LEQ(!uT6TW?FI3mj5XxjaIRbs~MHao;LUgTvI+4AYXAi=7t*V6-Tk!YXP96aEE!HVwqScX^QH zoxe|ateQd!Z1l7@dM6D*ilfNqc2Ylb__$gwNwPP$_A*-`ufS3621pf}xWCqh?rwxF z0AAju{*LyOWTbTv52px4yE3mENUyN5^=97S#SiE34 z&Kyw*K%>CREq?bbjnNleHWr%yC$tmjUT)Hj&ZrdDwCw>w;;N&#c}vI)hs!fV3MutN};+#v0YauHSeGp3%QBd8t~N zWIfo99QM}sIYDMR{+9ENaHoX-eMQ8Lrcpi!>L$4OCJ?IcrR$!(?diO_TUnWoVoj0HQG?Ty?ztDT{&udrq3WZ^swrp2LTK4fu&?ov zyt&TTnazn(amqj5?Lpa-G^l9Xlbf@n2;6;;(Q0?xhhZ!p!-L*Hk1y zsdH6U(3<4J7(A2)XBEq=rzTlYs*`ofFl03@_hhK!8s>Uqv40_wINT)Sj%R+?4Y#At z3DS-0*}4cplmDm7G+M6-#v*#|4jU7*E3Rc*Kp=;@`-tC;$)o@vcKEG-bJaxGoq|t( zJ-qS{+t~o-F|{Z~m(2$PeMJzr2C@;i!jqY%@Nc*uWlC7}ewKQm_sPzXm=$L`Kz8%g zmUkL>iEz{Ag9!ORbd(;_4zX=HM=_H5d6K9|2<2e(yTs5zeK#hckJ~3}mJw=vrDbS` ziFEUT{h0LTrl^wegH(yz3Ps|)e*6&q3(CJZ8z1zJqO3g6uOMsP=r!7oY($5wXVcv~ zn@O*CFM6l?1$bP@`0Qd{e)GM1xP;RxtpAv_{SK59(B%-ygnaRVp=*FH&c_g?q9yeM zCM=ko=PK(wrUzNJzcuFP^P7>Wa>t z7r?Cau^>{PSEA8A{s1y4*Z?~j)|~`-cyl3WhtmdE@Aw%T{+3t#Ux7&bqd|<=S zJdW`tGd-?gqH+jvc46GzSS{=r@RaCoimB6uh;LOo9qhGN%HGxy(?rV`W-r4ehGX3p z9+K>Hm5dWma8jAlMT#aPak%hOH@driWYqlvSR^w z^3@1sm$t5-G9mb{o^ur{f;9X#$7FIXSdqt5XIF$;B==HsuOrS z*q}ouPM=JwC7C+TYVs&UsfTS!G)YP%aMdF6B*Y^`W5AT9!VDyLl*QGQ3Y8b7`-3ON zoKyUGMd~D|PdiXbn=_xeU4jrthzWtg6{(B~=mfv8|L$UVb5oRK&%t4j5Z3@Gw2Q@W z{vII1-9xGjq>uiis@TIp%~zHo&HT6PywQGewt-Y-@|`sdYcsk54(WPrv%4 z4RJ#9wHtSbvl}HeG|wl1dH#a<{By2H^Mo)rAuCj`y+T<=*yN_*c!VM*FAwgg_M7jhUssWcui(IJ2H1D9W(NV zo2_!ob~QVtQv&!6v5n<#`(w#my*rmYR&!HXF!Fi+&l65 zVO91mkN&kERHO~Ugueh!;XRVmv0L!|PPYPT^QTcz$ z-@g#Xib@_dIQIX=%S6u5?U6c7jt~V;PkV}cyKFbE3M_#K$M!VHlZJq;}?pP zZI)*spPUxIp9T;`msRetJ}u@}UqGTGa!z}@!;J9%p76*8SNL8`bW*SeIRP%aAO?gD zW;gN+h+X`{@UfTRaud%^@NmA0Ldyb)`R8quzLl1wv3C(F#yA-f%9;lLyDFzZ`OBuO zi0GfV*MMO@C#C5IRtGPx`rPhTzFzmvQ5h|{q;ormmtaa0L)H8nIR2#l|8ham(m`K9 z6`+7Fkub6IZR#(eW1tYNi*5f8a+&?@o1M+}UkN6q|I0k+v8DWxM*1a%E%Z|izHwt# z;QyC(aQ@di*)_(J7H)wT6S`ynWgQrLA#0o`8guR+l8t3(SE6z-Yj1GAUESNB-B(th zP690tIOvF)_9FkU?@qU=0fkLBeFv3?zIVQRKm^XVwY`I~ zfI^5YyP*7UgiHGbl?AjmY^ZValf|T!fC=+G1I5{VPe8p`yS=>~lNx&$b-bkna#84g zuv5|yEt|N4W)CXZq*)VWo{OW}GTJGm*PPz^sY3ttygAf4#>>C0mpq&4aWXoXd7i6PD{BL1B}DjA<(6^yuzAkjlp+>0Y0a^OQ<*Z5fSoc&rbve@8Rlf)eT? zy-r_LiG8~cp^?XkP-PvE>Z;&^w6D)Q|01FxB+t-8bt4NLeyq^krof*Fiho|>?kk+@ zIa}y~{H>k-s$4DfWE+GGj<@{gY9|-&Z{X$SA*~_U~qK{nD zxA+VFqV7UL*xnXnnIlnS6P+|`#FEq!3TOIb#tNgkE*;eDnYMu$)CzmJ*(&7i2uIq0 zOoCOKK+$QptGAjqYLFwl8Qx=-Rk@Z%Z|XZvC{UA2G<}`#Dv;xur9$&Ft1Nk&x|&)* z1MuK?8<`#rv^Gt|8ec{#8@_(!bQL9!jk+ixt066ljJ8w1Dxf}#pJEo+Mn9MtV|Gy< z^6v^;7h1R-JI-a;o+Xc?y2-v37(=NJeQILy8st;O`0oh zE3C=2kFcO<5~Euk5t_Ltd;-ddi$Q^P9;%RTHN_~$&uMVOtqevAJ?fDpf@p1Y4bWTD z3RzlSa_^p}>(n!ntQ8m`-hgv}fw}@DuCD}K!ze(7-aE5cIJZI(F8R&lCza$m{Kl&7 z6Av&}2>3C<6z9aFVG3!n1~BR<=|c#EpPZ}|!bl-aQdkrd)=CR8&2L32C~09OKCs4N z9DAsGm-_2C4j6ZSMl~?DQ1?P=06gka*+AQE_~`OGaqDEMLrM}?fbN7DbiKt~|07sv zZt6QzASok!26Q5NkqzKBo>NL2jYY{c2V3Dl?p!)prdn*3e)sC2b7u>h(*I-U zT8P7^<|Xdt{Ry(C<`W0&%cWlm*;{GP4tz%y*}+*=_l?#RRRd}AsAc?QSLsf?=% za~B#Jj#KFBDV>6mK=!NMZVCxrLG~|XvnN?A3o4pD=c96>nZK{Q%4d5MK4GmTB4^mRQj-Vdg?l*mby4npJrlY`vp{Lt| zgHfgHzkwnHYzCR&zX98B7uaM9(EOP~>ByrJZF40=yQ?x5T6V=;;qM3$FW7Yq}#aIG-n0}DP^JOzf#O8s$8vS@hlsD2?vkRDT-8IS* z{qC>fANC;AQKT&s*LXk6ou5=lqqEi6}~>BwxLqj$?(v18uyH7I~}=ITV>Tl1BmOy_42v?$tE% zZ2|KX4RGwWE`HNht@!krj%yep2Y8^_h{TDRREx#kVoajjtJvk0AILo-VK>( zCCQ6LgsUhhbPuTuc(6kgv@jI_Ocd{A2$NhG%&bW&Vd)YcGLIxL&@K&Y77;zf4ihgk zaQ1N-y0ic!jt{xfNep%|`G!hKabRbYA%q!PK7;oet%7RLk)dlAGjaYXRM*;7h86y8 zCB1>}{9(1sW!{IdZjHN%VnJiZ10)mS+bh*a?0Ch`A;Y+QISxb*TLU>tychkYh= zaiCvVIzcRr2>yn+h!kX0DBI;EdS|A77B;1dbP-7O=SSd-Iu_cG610&3puvV?_ht=; zy(`Cop{S<)!P-T3E8%`+7rZ*#_3dnPv9!Zk8Bg>)z@6@G&g(_*iSdv*>-OsK5x74C zB$}Dg-;lpHJlMNaMg%%`JpzWZd9GeOfNPS;(VW$n$EW!5*&aKYYj$Rn&pfr~2bCAd=AN0F-X1hweded&53Sz%O2oo*=|F&Rya z5;?I>$Z=RVACyxab%3P%bRCzPq2c#Mu?p9d3dn3nyq(Pm#KyD8@jqS8xZe*;&TrC- zC-)eseuN8_`#i&u=3-5k4=R73hR^+%FD@8H60wa?Qn+M6)cyCuU>2cd8ruG0gkHA; z2-VO5NyLqjIw+Ec&MKKC>i-<&TMg?X_DC#A$}M753t3Anff)mDtk|zF$Lsa1bs-Bq^TyUeeCUN7o>4 z{n=!sKTm>M_9&WjOZL=9<6S64izBKptA?h%uX~f>cIf=QD7l2@+6O>fd-`y1+mR*l z47r}-yaTLHDvoT%Yl+e?6B0#f8R?rH9&jibMrbs0d0@LDnVxMF5ZOH~kFxg2x#oA0 zY(>Il=g!Yhrg)atEhb}U9Fs>%A0Ifi*qxMBWw}u>{2SWlLd85jkitiu>bn+>sCU)< z((uu_DnD?UiGDXzK^6gR4icdb$CNMzp(jBUPq2tRNQl+Np|{(_jVeHB4ptep7%PBZ z$EElGpMy48qxfLh#a82se>v-tEh3XpNg{0iErfD7Py%n`Er5zsy>IC-@l9VaSLDC5 zOMI<+e4daa%e2V&)TKIEvz4j(InD!?pX%~PSZ59n0BBKdHFbPad|DO9JQ->_^H%xi;rhk8VxKFf_ z%L@MrOk7IptnTu-th%z{UhayqTt1PL%CD4LO^{BPE@i=UqYRlq9|1(mrv7wfDP^s7 zFVnMVAJNaop<@KOvGmAThNhM5-r&;%Kf7svo;506cM8k1D4Gzc&C`LlNP*OS;#2R*bbQ~ ztTl5I-EG>5qLMW9r}$tPy!q`RF?cH}-XAYe4pLJqNU>a&%vS6KRZxpWF9?)%U7<<@ zjFUJ?Pz`qpfEkYz4K+;xoR%20(;sau_T3Kf8woHh=-F8G*)eZ)w$8vZ3EgI}0m>>k z1iJ9)f=G~e#IiP6%D_s5=c7M_ATkmSpx-=0g2KYkA30&MQiSBFW%Y6QSyj;W@ZZ-($%kx)>dMcg}aVy9Xe zE8XPSXy4HYJ6KEc(m*CeNmW49>?T7aAIu>i6#r6d{4=3RveIcMBX30c?{6uxUsoX-*p?wvjiyzXmi+A~S{3P?57?-1x? z>ga3q-sPELhG58R)L-N_2gprR+I=Ylc&0`|_EvVD2nGGe5q~wkkTz|T!Sd3`z)Nb_!@;9d#?zy|a zp@453QzTzJhVH?CQrBa#+Vk=!R;?Y2{jF?&+gyG$esG^i3|A4C^1!M%8}YaD5-MWe zW9Mpu)acB>g_-N;rw@YuMP)7w+%8FKBp^{P>+q)~FIx^kh>Vjl5s{`^07>v57`D*` z2aGx(B*=iAvIybHUv^efv&YiIRnL)UGEC_#%@*XK*pGKqx%^s5xes$WrhLrEA!C(C z&eDJptxcy1Wz%Q$wZ`qiAUM=9NT%(~{9Aj=&T`sz6sImPYIAF=!>ATsVk@d5t6poH zgx5L#$ll>&t8TBdw4PeUupcH}Z){el4mijHw0Hic_@a)y*B|b9%PG-j9Z4p9>Mg6A z>uYA@qajE60X(cHmFfN2>$6m*I5pT^(|?e?d)q3n7kk&sE%iFv(sK=Qtr8oNsL1@M z>u+7O5YV3-^#WxcU9`z;kPb4Jj%^sMUpE-EHidQ{^0zoRNmc^prYGh}>qr&MB7pSP zs_VPgI=o2Ek$-wQ4x_&49=-+<#GFtka>*;(7$Y^1Og%ckzWj7h?tv;0G%CdR>G zd5@57i>CK8=H<8K{w;F>-v{>_Z!QA8t$?Y|dzc^Pd!zZE_s`BY{5dhex9In|&#%42 zLr{b8A&oJr&9CfxfS>|FE-`Smr)1o^|JS1NT(k^R-!1wrRQ=w3)*};X$pFbtAaMovTB(2b+gN7fI8PgRj7rD(6St z*DAQgTql}u7DO*4=5EfYi7m#(W?W7pf$Di3A@;Fm(jrS+z$`860npZ5SsjcLTPrn( zj7gi$&x0d{>1q5L0RW1m6?^ zd*P3J=0c=1!|lY7hnS;5kRYRH?JN*w`#%A85sB`9+=-KW(OX^Ab-Dzwi*cNG z`3yjdV9-{duYy2(t|t79ga7=0NAOkr^D7$(t(V$}v^uLtOPovIy!6}Jfg0A+H==d= z?&h^vpd|TLh{z+3Z{fW$=ZYeekUiXx?%W_^;^-+u#h#X{J32n>8JtW(_6#kD?PKXaS)g*s1vXIq?oxlV6| zh34d^o^D{oFj*X)u|03%6;%}HN|8sao-pTF5l}Isvvj^E!#SOlJtxR5@_HOS%?bDD z%XFI#Y7a!|Htqf|Z&P1Jol`@qO^=t40{XX{JPp3HUDBoEcOdOYs zr$@L4*3J zd#eoM0C^5Oy}MY%CG+7vUDE@}_WJWYKK<+U(b4M{PfmxoFY;k*u%FK_XTvFSg!sxR z=1YhBr{Cs_{Pu8sHHoR#^>DtJeS5S!m|WzK*~x5>&xYgcM~4F>Jj2De$LMuB9p$%_ z|L~_z+5U&PWF|>}XOlk;_mBHiwya!Ad8+-h-T5U6@Fp<2OXVG*1v|x1{>;2X8ZgDM zuWwSF-Otw_h65xtSP(~1dz1+6AT`<%hTb()cn<+Yv3ie=~y zZ!^jB@#SOyY*%pYyePmJ^dWx|_mWfYcQ$!H9-?+mZ}RTzu!RbHuTBr&J~=%*os6#+ z!=2M-J23R!s6X4_MX?+GaJ={s-*y%-}xsFUK4dU&=^A{)2pS(Igo{T3# zR&@M1zHAUNZRmhT(kB)*4R@0;SP-lUBc&_|V@-)LRA)_+P!1`y(%4b7ND=YXbxTlj zjU=IeNQijpUxg%X=q)KhFX!#b{~~An77S?-zf)p=MHi-oO{VzmW6rbp&)n}k~-;iS2^6IH(wpe)F!?QgVJ7o zl|o^A3Z1+=d3|#5q7{_NDL~`okh-3;L=^y8jb*>bLgl*;# zDy_MHBg@<|SfPPCE|at&iR7V$Jkm;G_y&Wpv;>3bS2B=+GF>UID-60H-xZI{29N9( zJi5P=PHa*eY?9wybqN+hoyXsGr9JA|eDUn2KLc{~$*5i}Nw5wjyka~=hTpg@D4|z>lS+9ak|Em(Htwy^Z=X!BJWI%8MFkyT zqetv^IKG>+DTqB8P-_Fwo-f!apU?Au?)oEk zO?&@n6EFJH>1^_~D9>h-FEi4L4aZl*F$~1=kFN{ZWj-7Y34yD{vYez}%=(vkyd6u$O-zkA1{WhX>ko#P{n6t! zc`Mesp9y8+q#&{4hOTi#tho`?ni`)zygxeq3!S|KVK$hde}EY_=+ZQUi>hz07A?ye zVoeN%1&TBx3*I9k;ZV z(cAH1ldsr^_LU0$D~GA#FUVukB)Z`;N>uT`$l)=A=r>ea+|c)^e|s_LBcihDkj-x3 zAAJ;WhHN??vK(LJ2)GtS42Ele+}`%1jh4Zc?IIvPWBcqmJ79c$e8#ZMBWq-3yy9O_Fztq%7;hJVmU6`7F1+rmk-C$~eI>?G8i@k(h z{$!&x(rAn8k`?{|@^#$HKy+J`C{qL3moVmHjq&T(Z=WALhv?@Yn^q;hE6DtNfowVH zXxA|5L|o%?L>o<+L={+D19mx^UktBH6FOb3W}&_1MSMh8^E+68mI;ge>!Px5h&NV` z2gCVv)c;ngvAlQFUwj#V4lsueM}wS7B_mak+%6N@dv)~TXs^DAZAwDi!zUsBCz22> zVj7Z=+U}Vu=dYrigV<^g!d=${cA0)yBpn7RY*YHt3hFBJ*w}6L+w8Ws%I}hN$jua1 zx$h?(Z$Fxjph^gUMEj68OT58&V<;3YWqp9Xqr;kg? zmPKTF6}VQfGn86XpwzNHK?zsR>0D`sI@EmNs5^wZ3SoWvDUq?XjDEFu`0~SxC&#C|$4%&oho5}-CJhmT zw57kvp}#cZTvefes!cs}EQx33s9#Z`J6x^`m#@KPOL+MbU@y-;9G@Jax4lNdWOurm zpMh)#*~X5RJd+As3bID8P{l02oi4u7jsTXSCFCl)bVe@yjl~K%Q{R2jVT~NhE9CZ% z_FnFuQCzT_xT#+i_PlCZ?C|9xlKxz-=9{Lr^@3nm!mOl!MYnaJ*;s6I(XtZ!YybG* z?X$Rk@tY9o2lqwSxDa%-5Ts<5e)Qy=sDe{lVK*(q%4%%)^6Bj8-R__J%l8}n7ruN- ze*{r_`Q+74y?hF7FQ2OKvo*X0*{Y3MDQs^a^5s(-dL{FDS$X;7w|PC;B7WVn)Q#!pBQ+!NnADk)8;FXC|eM{-Lc}v+WeOsOg zt~ZHdwM%DH8eRjH90dgI5{;oK0G!zF<~%eVP-I7WnwooH8Eo~V`eopIW;vo zGc!0jH8z(yNCrrMH$FZJb98cLVQmU{+BMi`P!(qs#_{)DF4u-&K~S+^7eTRJ1ySsP zT~Sm-QN#i^?5KdK*s+%=VDEq(E7%p2$@h#iiIZ=c%p{XZVkYV&8FTl4Ucdb2%(Lh2 zUUv7M-4|nw`TW^sT(^xE`=JZR;TW8V<1ra0VG2&c88{7pQ!x!^;~dPuOk9KuFbfxp zF+s+-jA?0%NUAusFZUA@QeDL>+()F;GlOmF)?lrwMftq@|e3C3b5VdAOMBt~O~! z(oW1bUwT@TbP)4hsZH9Dbi#GGhCCeGVte%M=f^VpsSb2@#toQ@`M6Qc&ndREBT0mq zzw&IN$Lc2HinEFCCsHKvlTEsibQfupVUw;TJw$qc53)%FNl%fXg*NF%(o1C0-!_T# zC4203{EB)n?#F|8n5|LNM^QUZU!w2{XALCPej{Wf^d+SYIi*HB_ z3cASky!oLU{n3MUupc(1KL%q-g0UHfU??`n7O21Ffz)Bx3iY`0I24ED2>RowN8%_< zz(h>K(HKkrAnJvfg^O_sF2!Z2GjK4*;}G(H99)4bQNP1>+(oB;&plX-`|toB!XtQ$ z{$bR|QNQ60yoGnL3iZnB8F+w?uo`RdF%IS)>q@_#tC1v;*d2RdH0m#<7oDT1^$yqj zwlDh}LOmA8<3#!ss3+lMOvWiV6;p5;{d#qVku1igxEynE1+Jt&2}hIYFLy2G;s&;V z>I|7pJ%+k9w!yX-j_t7{cBX$Ubp+}h>xpwP8Kx6?V3dMED2V%&%N1YAVtHcUgE2HDgPsq=9Y z7NSmmy-&8_RxHA8xD$8NS%Z3qr;}g8D|i*lu>vdU)O+|EUdNlL&yFYj6^&4T-@f!* zwZK-`7TaS-?1J6s&!CQ?enveUb=v48(fN=_k6wjLk|fmM^%xw76XAg?hw&cmNOK5j=`I z=NF?Mu>|#v=oD9^9Y@)UdN1jJ_uqlLa1ZXq{dfrVSUN?Iv9}H+`bM-9_1UEN(jL?^ zqffeG+>871ARcCKov3s05?)5V_qt#=)b~1lj_6ZTpB?>CCv9D10lSLA^f%unC4> zOKeU5I_hxjfSu5wA9}DJ{kgcFBoKo!6j$LI)IIBSVFUF>%)@*vz)kdA#d6$;6{s`e z8eYd6sPCfsT+n?s#HQFB!>|qNy{b>od@MjcdZ9>8ma|sP^|ycfMOGI27JUgX;}yJ# zSFs!`uoAD~b-aO(@D|>G#yePrckv$H#|QWjpJFxE;A7ND_!4!FJx85FIz3+VSLk1) z>w-7p@t(Eo@xB;<9@MqAu8Vcmtn0^ysB2qY5$pO_SFFJpg1Q3Lb*rx6biLUgb(N

(b!w0DBHOTyLLJr|Jt$SiR0-&`ggh5UC`8sd{gd7?FDkPV%UQ zO=3l=Yn|l#Uu`l_ 审计对象：`src/mcm26b/training_ui/`、`src/mcm26b/process_control.py`、 -> `scripts/train_hrl_high.py`。 -> 触发问题：「无法暂停」「频繁出现 os 无 killpg 等」。 -> 结论一句话：**这两个现象同源——训练前端的停止/暂停路径是纯 POSIX 设计； -> Windows 上 `os.killpg` 根本不存在，异常又没被兜住，所以点了停止等于没点。** - ---- - -## 0. 结论摘要 - -| # | 现象 | 根因 | 严重度 | -| --- | --- | --- | --- | -| 1 | 点「停止训练」返回 500 / 无效果，训练照跑，于是「无法暂停」 | `TrainingJob.stop()` 直接调用 `os.killpg(os.getpgid(...))`；这两个符号**只存在于 POSIX**，Windows 上抛 `AttributeError`，而旧 `except` 只接 `ProcessLookupError/PermissionError` | 阻断 | -| 2 | 控制台/日志反复出现 `module 'os' has no attribute 'killpg'` | 同上：错误最终落到 `_UIHandler.do_POST` 的兜底 `except Exception`，返回 500 并带出 `AttributeError` 文本；每点一次复现一次 | 阻断 | -| 3 | 即使把 killpg 换成 `terminate()`，停止后模型/元数据仍然丢失（Windows 上） | Windows 的 `os.kill(pid, SIGTERM)` 语义是 `TerminateProcess` **硬杀**，训练脚本里"收到信号后在 step 边界保存"的处理器不会执行 | 高 | -| 4 | 界面根本没有「暂停」能力（POSIX 也没有） | 原设计只有 start/stop；且 `SIGSTOP/SIGCONT` 在 Windows 上不存在，**原生暂停在 Windows 上无法实现**，必须改成协作式 | 高 | -| 5 | 子进程进程组隔离在 Windows 上失效 | `Popen(start_new_session=True)` 在 Windows 被**静默忽略**（CPython 的 Windows `_execute_child` 形参名就是 `unused_start_new_session`） | 中 | -| 6 | subproc worker 在 Windows 下可能抢在主进程收尾前退出 | worker 只忽略 `SIGTERM/SIGINT`，没处理 Windows 的 `SIGBREAK`（`CTRL_BREAK_EVENT` 的落点） | 中 | - -**修复原则**：把"暂停/继续/停止"从**信号语义**改成**控制文件语义**。 -控制文件在 POSIX 与 Windows 上都只是普通文件 IO，行为完全一致；信号与强杀退居 -兜底。这样 Windows 上暂停、继续、优雅停止、落盘保证全部成立。 - ---- - -## 1. 失败链复现（旧代码） - -``` -用户点「停止训练」 - └─ POST /api/train/stop - └─ TrainingUI.stop() → TrainingJob.stop() - └─ os.killpg(os.getpgid(process.pid), signal.SIGTERM) - ├─ POSIX ：成功，进程组收到 SIGTERM，脚本优雅收尾 - └─ Windows：AttributeError: module 'os' has no attribute 'killpg' - ↑ 只 except (ProcessLookupError, PermissionError)，接不住 - └─ _UIHandler.do_POST 兜底 except Exception → HTTP 500 - └─ 训练子进程完全没收到任何指令，继续跑 → 「无法暂停 / 停止没反应」 -``` - -关键代码（修复前，`src/mcm26b/training_ui/server.py`）： - -```python -def stop(self) -> bool: - with self._lock: - process = self.process - if process is None or process.poll() is not None: - return False - try: - os.killpg(os.getpgid(process.pid), signal.SIGTERM) - except (ProcessLookupError, PermissionError): - process.terminate() - self.status = "stopping" - return True -``` - ---- - -## 2. 根因逐条（含证据） - -### 2.1 `os.killpg` / `os.getpgid` 是 POSIX-only - -CPython 只在 `posix` 构建里注册这两个函数；Windows 的 `os` 模块没有它们。 -访问会得到 `AttributeError`（不是 `OSError` 子类），所以旧 `except` 接不住。 - -### 2.2 `start_new_session=True` 在 Windows 被静默忽略 - -`subprocess.Popen` 接受该参数，但 Windows 的 `_execute_child` 形参名是 -`unused_start_new_session`，也就是**根本不读**。要隔离进程组，Windows 必须传 -`creationflags=CREATE_NEW_PROCESS_GROUP`。 - -### 2.3 Windows 的 `SIGTERM` 是硬杀 - -`os.kill(pid, sig)` 在 Windows 上只有 `CTRL_C_EVENT` / `CTRL_BREAK_EVENT` 是 -"软"信号；**任何其它值**（含 `SIGTERM`）都会走 `TerminateProcess`。因此 -`scripts/train_hrl_high.py` 里的： - -```python -for _signum in (signal.SIGTERM, signal.SIGINT): # 旧代码 - signal.signal(_signum, _on_stop_signal) -``` - -在 Windows 上形同虚设——处理器注册了，但没人会用它。这也解释了为什么"停止后 -模型没保存"。 - -### 2.4 Windows 没有 `SIGSTOP`，暂停无法用信号实现 - -暂停（挂起/继续）在 POSIX 上可以用 `SIGSTOP/SIGCONT`，但 Windows 既无这两个 -信号，也无等价的"无侵入挂起"用户态 API（真正的挂起要 `SuspendThread`/调试 API）。 -所以**任何跨平台暂停都必须是协作式的**：由训练进程自己在 step 边界检查标志并 -让出执行。本项目原先没有这条通道，界面也没有暂停按钮。 - -### 2.5 `SIGBREAK` 未处理 - -Windows 的 `CTRL_BREAK_EVENT`（发给进程组的软信号）在 Python 里落到 -`signal.SIGBREAK`。旧的 handler 安装与 worker 屏蔽列表都只含 -`SIGTERM/SIGINT`，Windows 下等于裸奔。 - -### 2.6 异常类型没兜住 - -`except (ProcessLookupError, PermissionError)` 漏掉 `AttributeError`、以及 -更一般的 `OSError`；`_hard_kill` 里 `signal.SIGKILL` 在 Windows 上也不存在 -（`getattr(signal, "SIGKILL", None) is None`），同样会 `AttributeError`。 - ---- - -## 3. 修复方案（已实施） - -### 3.1 新增 `src/mcm26b/process_control.py`（零第三方依赖） - -| 能力 | 实现 | 作用 | -| --- | --- | --- | -| 子进程隔离 | `spawn_kwargs()`：POSIX `start_new_session=True`；Windows `creationflags=CREATE_NEW_PROCESS_GROUP` | 前端 Ctrl+C 不误伤训练；Windows 也能组杀进程树 | -| 控制协议读写 | `write_control()` / `read_control()`：JSON + 同目录临时文件 + `os.replace` 原子替换；损坏/缺失一律视为"无指令" | 两端统一的暂停/停止通道 | -| 强杀兜底 | `terminate_process_tree(process, grace_s)`：先等宽限期；超时后 POSIX 走 `killpg(SIGKILL)`、Windows 走 `taskkill /F /T /PID`；所有 POSIX 符号用 `getattr` 探测 | 彻底移除 `os.killpg` 无条件调用；返回"是否动用了强杀" | - -`IS_WINDOWS` 是运行期常量而非导入期分支，便于测试直接 monkeypatch 模拟 Windows。 - -### 3.2 控制文件协议 - -路径由前端生成并随命令下发：`--control-file `（默认落在 -`output/hrl/ui/control/.control.json`，`output/` 已 gitignore）。 - -```json -{"paused": false, "stop": false, "seq": 1712345678901234567, "ts": 1712345678.9} -``` - -- 前端**只写**：`pause()` → `paused=true`；`resume()` → `paused=false`； - `stop()` → `stop=true`（同时清零 paused，保证阻塞中的暂停循环能被唤醒）。 -- 训练脚本**只读**：`_ControlPoll` 每 32 个 step 抽样读一次盘（避免每 step 一次 - 文件 IO）；暂停时在 `_on_step` 内阻塞轮询（`poll_s=0.2`），进程存活、显存仍占用， - 但不消耗算力。 -- 墙钟上限 `--max-wall-s` **扣除累计暂停时长**，避免"挂起一小时回来直接判超时"。 - -### 3.3 训练脚本 `scripts/train_hrl_high.py` - -- 新增 `--control-file`；`_ControlPoll`（纯标准库、可单测）+ 薄薄的 - `_ControlChannel(BaseCallback)` 接到 SB3 回调链**最前面**。 -- 收尾信号集合扩为 `_stop_signals()`：POSIX `SIGTERM/SIGINT`；Windows 追加 - `SIGBREAK`。worker 屏蔽同一集合，保证主进程 `vec_env.close()` 之前 worker 不先退。 -- 落盘保证不变：停止后仍写最终模型 + 元数据，`stopped_by_signal=true`， - `stop_signal` 记 `control_file` 或信号名；异常仍落 `*_emergency.zip`。 - -### 3.4 训练前端 `server.py` / `static/` - -- 新增 `POST /api/train/pause`、`POST /api/train/resume`；`stop` 语义改为 - "写控制文件 → 宽限期内自行退出；超时强杀"，强杀在**独立看门狗线程**里做， - HTTP 请求不会被 15s 等待卡住。 -- 状态机新增 `paused`；`_sync_control_events()` 用训练脚本回显的 - `paused/resumed/stop_requested` 事件校正状态。 -- 快照新增 `forced_kill`，前端横幅区分"优雅停止"与"超时强杀"。 -- 前端新增「暂停训练 / 继续训练」按钮、暂停横幅；停止确认文案更新。 - -### 3.5 停止/暂停时序 - -``` -pause: UI 写 {paused:true} ──► 脚本 _on_step 读到 ──► 回显 paused，阻塞轮询 -resume: UI 写 {paused:false} ─► 暂停循环退出 ────────► 回显 resumed，继续训练 -stop: UI 写 {stop:true} ────► 脚本 step 边界收尾落盘 ──► 退出码 0 - │ - └─(宽限 15s 未退)─► terminate_process_tree 强杀；forced_kill=true -``` - -### 3.6 急停（强杀）也必须可恢复 - -宽限期一到就强杀，训练脚本**没有任何 Python 收尾机会**——不写最终模型、不写 -元数据、不写 `training_failure.json`。如果磁盘上只有 `--checkpoint-every` 的稀疏 -历史 checkpoint，急停就可能丢掉上万步，或者在"保存途中被杀"时留下一个打不开的 -半截 zip。为此加了四条保证： - -| 保证 | 实现 | 效果 | -| --- | --- | --- | -| 起步基线 | `recovery_every > 0` 时，`learn()` 之前先写一份 `_latest.zip` | 训练刚起就急停，也有可加载、可 `--resume` 的产物 | -| 滚动恢复点 | 每 `--recovery-every`（默认 2000）环境步覆盖写同一份 `_latest.zip` | 急停最多丢 `recovery_every` 步，而不是 `checkpoint-every` 步 | -| 原子落盘 | 模型 zip / 旁挂 json / `resume.json` 一律「同目录 `.tmp-` + `os.replace`」 | 保存途中被强杀，目标路径仍是上一份**完整**快照；不会出现半截文件 | -| 权威恢复指针 | 每次落盘刷新 `/resume.json`（model / timesteps / 增量语义 / 生效 PPO 超参）；前端 `/api/train/state` 暴露为 `recovery` | 无论怎么停，都有一份明确的"从哪继续"；界面一键「用最近检查点续训」 | - -前端在强杀后还会补写 `/forced_stop.json`（`job_id`、`pid`、`reason`、 -`recovery.model`），因为此时训练脚本已经无法记录任何事情。 - ---- - -## 4. 平台兼容性矩阵（修复后） - -| 能力 | POSIX | Windows | -| --- | --- | --- | -| 启动子进程并隔离 | `start_new_session=True`（setsid） | `CREATE_NEW_PROCESS_GROUP` | -| 暂停 / 继续 | 控制文件（协作式） | 控制文件（协作式） | -| 优雅停止 + 落盘 | 控制文件；`SIGTERM` 亦可 | 控制文件；`CTRL_BREAK_EVENT`→`SIGBREAK` 亦可 | -| 超时强杀进程树 | `killpg(SIGKILL)` | `taskkill /F /T /PID` | -| `os.killpg` 依赖 | 有 | **无（不再调用）** | -| 暂停期间是否算墙钟 | 否（扣除） | 否（扣除） | - ---- - -## 5. 验证 - -- 新增/更新回归测试： - - `tests/test_training_ui.py`：控制文件协议往返与损坏容错、`spawn_kwargs` - 平台分支、Windows 强杀走 `taskkill`、**无 `killpg` 环境下强杀不抛异常** - （直接复现旧 bug）、暂停/继续/停止全链路、控制文件被忽略时强杀并写 - `forced_stop.json` 恢复指引、`discover_recovery` 的 - `resume.json`/扫描/悬空指针三分支、HTTP `pause/resume` 端到端。 - - `tests/test_hrl_checkpoint.py`：控制文件状态机单测；原子保存失败不破坏上一份 - 快照；JSON 原子替换；历史清理不误删滚动恢复点；起步基线可加载； - **强杀急停后 `resume.json` + 滚动恢复点仍可 `--resume`**；控制文件停止后 - 模型/元数据落盘（`stop_signal=control_file`）与暂停→继续；POSIX 信号路径 - 单列并加平台跳过。以上集成用例都用**真实训练脚本**跑 subproc。 -- 手工验证：`--recovery-every 32` 短程实跑后强杀，`resume.json` 指向 - `q3_ckpt_latest.zip`；以它为 `--resume` 目标再跑 32 步，元数据 - `actual_timesteps` 从 64 变 96、`timesteps_semantics=additional`。 -- 全量测试：`pytest tests referee/tests` 全绿。 -- 全仓库 `pylint src scripts project tests referee main.py` 达到 10.00/10 - （口径见 §7）。 - ---- - -## 6. 残余风险与后续建议 - -1. **暂停粒度** = `poll_every` 个 env step（默认 32，约 0.1–3s 量级），不是即时； - 若要更快可调小 `_ControlChannel(poll_every=...)`，代价是更频繁的文件 stat。 -2. **宽限期固定 15s**（`server.STOP_GRACE_S`）。大模型保存可能更久；但因为落盘是 - 原子的，即使宽限期到点被强杀，也只是"新的最终模型没写成"，**上一份滚动恢复点 - 与历史 checkpoint 完好**，不会出现半截文件。需要更长可调大该常量。 -3. **急停的损失上界 = `--recovery-every`**（默认 2000 环境步）。要更小就调小它； - 调到 0 表示放弃滚动恢复点（此时只剩 `--checkpoint-every` 的稀疏历史 - checkpoint 与起步基线）。 -4. **Windows 强杀依赖 `taskkill`**（系统自带）。若被安全策略禁用，会退回 - `Popen.kill()`（只杀主进程，spawn 的 worker 可能残留）。 -5. 更严格的 Windows 方案是 **Job Object**（子进程随 job 句柄一起销毁），但需要 - `ctypes`/`pywin32`，本仓库基础依赖刻意保持轻量，未引入。 -6. **前端进程重启后**，内存中的任务对象没了，「用最近检查点续训」按钮不会自动 - 出现；但恢复信息都落在磁盘上（`/resume.json`、`forced_stop.json`、 - `checkpoints/_latest.zip`），在表单里填同一个模型输出路径并勾选 - `--resume` 即可继续。 -7. 界面 `--device mps` 选项在 Windows 上无意义；`auto` 会回退 cuda/cpu，属既有 - 行为，未改。 -8. `_ControlPoll` 的暂停循环会占用一个 Python 线程的调度（每 0.2s 一次读盘）， - 长暂停期间 CPU 占用可忽略。 - ---- - -## 7. pylint 口径说明 - -配置写在 `pyproject.toml` 的 `[tool.pylint.*]`，目标是 -`pylint src scripts project tests referee main.py` = 10.00/10。 - -**改代码修掉的真实问题**（不是 disable）：未使用的 import/变量/参数、重复字典键 -（`reward_version`）、`value != value` 判 NaN、`__dataclass_fields__` 私有访问、 -`raise` 非异常类型、`pytest.importorskip` 之外的赋值不确定、过长行、无谓括号、 -`len(x)` 判空、循环闭包变量、`subprocess.run` 未显式 `check`、`os.path` 循环导入等。 - -**有意 disable 的类别**（每条在 pyproject 里写了理由）：项目刻意的懒加载 -（`import-outside-toplevel`）、防御式宽异常（`broad-exception-caught`）、测试访问 -内部状态（`protected-access`）、pytest fixture 同名（`redefined-outer-name`）、 -无头环境不实现 `gym.Env.render`（`abstract-method`）、以及静态分析的已知误报 -（`possibly-used-before-assignment`）。 - -**注意**：`good-names` 不覆盖 `redefined-builtin`，stdlib 回调签名里的 -`log_message(self, format, ...)` 只能在对应 `def` 上行内 disable。 diff --git a/docs/training-ui.md b/docs/training-ui.md deleted file mode 100644 index d55c5bf..0000000 --- a/docs/training-ui.md +++ /dev/null @@ -1,160 +0,0 @@ -# 模型训练前端（training-ui） - -本地回环 Web UI，用于**实时观察一次 HRL 高层 PPO 训练**：子进程状态、步进进度、 -参数收敛曲线、逐回合记录与完整 stdout 日志。设计与评测前端（`referee/` WebUI） -同构——同一套暗色面板语言、同一个「只监听 127.0.0.1 + 静态白名单 + JSON 轮询 -API」模式——差别在于本页托管的是一个**训练子进程**而不是评测会话。 - -## 启动 - -```bash -uv run python -m mcm26b.training_ui --ui-port 8036 # 打开 http://127.0.0.1:8036 -uv run python scripts/train_hrl_ui.py --ui-port 8036 # 等价薄入口（支撑材料命令） -uv run python -m mcm26b.training_ui --print-schema # 只看配置契约，不起服务 -uv run python -m mcm26b.training_ui --print-preview # 只看离线示例快照 -``` - -`--no-browser` 阻止自动打开浏览器；Ctrl+C 退出时会先停止训练子进程。 - -无训练进程时，界面顶栏「离线预览」可渲染三组合成快照（上升 / 平台期 / KL 过猛）， -用于检查曲线、指标卡、时间线与日志着色——预览不启动任何进程、不写任何文件。 - -## 界面结构 - -| 区块 | 内容 | -| --- | --- | -| 顶栏 | 任务摘要、进程/进度/墙钟/设备/连接五个状态 chip、刷新·预览·暂停/继续·停止 | -| 执行进度与进程 | 步进进度条、剩余时间与预计结束、PID/吞吐/退出码、PPO 阶段时间线、完整命令行 | -| 训练配置 | 由 `/api/train/schema` 生成的五组表单 + 四个预设，运行中锁定 | -| 参数收敛状态 | 判读 chip + 判据说明，每个指标一张卡片（最新值、增量、迷你趋势线、异常高亮） | -| 曲线 | 主指标（平均回合奖励）+ 可切换诊断量，悬停十字线读数、KL=0.05 等参考线 | -| 实时运行日志 | 关键字过滤、错误/进度 JSON 开关、自动跟随、最近 4000 行 | -| 逐回合记录 | Monitor CSV 的每回合奖励/长度/时刻 | - -## 数据来源与契约 - -三个端点都在 `src/mcm26b/training_ui/server.py`： - -| 端点 | 说明 | -| --- | --- | -| `GET /api/train/schema` | 字段/分组/预设/指标字典，**与 `TrainConfig` 同源**（`schema.py`） | -| `GET /api/train/state` | 快照；支持 `log_tail`、`metric_limit`、`after_metric` 游标与 `preview=1` | -| `POST /api/train/start` | 允许只提交部分字段（其余取默认值），校验失败返回 409 | -| `POST /api/train/pause` | 写控制文件请求在 step 边界挂起（跨平台） | -| `POST /api/train/resume` | 解除挂起，从暂停处继续 | -| `POST /api/train/stop` | 请求优雅停止；宽限期（`STOP_GRACE_S`，默认 15s）后强杀进程树 | - -暂停/停止走**控制文件**而不是信号，原因与平台差异见 -[`training-ui-windows-audit.md`](training-ui-windows-audit.md)：Windows 没有 -`SIGSTOP`，且 `os.kill(pid, SIGTERM)` 会退化成 `TerminateProcess` 硬杀，训练脚本 -"保存模型再退出"的收尾分支不会执行。控制文件在 POSIX 与 Windows 上都只是普通 -文件 IO，语义完全一致；信号（POSIX `SIGTERM/SIGINT`、Windows `SIGBREAK`）只作兜底。 - -进度数据的两个来源，服务端都已解析： - -1. **progress JSON**（主来源）：`train_hrl_high.py --progress-json` 每若干环境步打印 - 一行 `{"event": "progress", "timesteps": ..., "ep_rew_mean": ...}`，字段最全； - 热身阶段间隔会自动缩小，短程 smoke 也能看到完整曲线。 -2. **SB3 verbose 表格**（补充来源）：解析 `| key | value |` 表格，用于补 - `clip_fraction`/`learning_rate` 等诊断量。表格按段分组缓存、等本表的 - `total_timesteps` 到位后统一打横轴刻度；与 JSON 同一步数的表格版本会被跳过， - 避免重复点。 - -`timesteps` 是整个前端的横轴：服务端不伪造刻度（缺刻度的表宁可丢弃，截断的表 -按末尾已知刻度兜底并在收尾提交），前端不再自行编号。 - -## 收敛判读口径 - -`summarize_convergence()` 只做**可解释的启发式判读**，不替代正式学习曲线分析： - -| 状态 | 触发条件 | -| --- | --- | -| 等待首个 rollout | 尚无任何指标 | -| 预热阶段 | 进度点 < 3，无法判断趋势 | -| KL 偏大，策略更新可能过猛 | 最近 `approx_kl` > 0.05（判定优先级最高） | -| 价值函数尚未拟合 | 最近 `explained_variance` < 0 | -| 奖励未形成上升趋势 | 最近窗口奖励斜率为负 | -| 已趋于平稳（疑似收敛） | 窗口对半位移 ≤ 逐点噪声、变异系数 ≤ 0.15、极差 ≤ 15% 量级 | -| 收敛趋势良好 | 其余（斜率正向） | - -平台判据特意用「对半窗口位移 / 逐点噪声」而不是单看斜率绝对值：慢速但持续的 -提升（斜率很小）不会被误判成收敛。所有判据数字都会写进 `notes`，便于人工复核。 - -## 安全边界 - -- 只监听 `127.0.0.1`；静态资源是固定白名单，不提供目录遍历。 -- 命令由服务端以**参数列表**构造，不经过 shell；字符串参数只拒绝控制字符。 -- 同一时刻只允许一个训练任务；暂停中的任务仍算「在跑」，必须先停止才能开新一局。 -- 暂停/停止走控制文件（`--control-file`，默认 `output/hrl/ui/control/.control.json`）， - 进程组隔离按平台选择（POSIX `setsid` / Windows `CREATE_NEW_PROCESS_GROUP`）， - 超时强杀按平台选择（POSIX `killpg(SIGKILL)` / Windows `taskkill /F /T`）。 -- 产物路径限定在仓库根下解析，模型输出沿用训练脚本自身的 `--force`/`--resume` - 语义（两者互斥，由后端校验）。 - -## 暂停、停止与落盘保证 - -「暂停」是**协作式**的：前端原子写控制文件，`train_hrl_high.py` 的 -`_ControlPoll` 每 32 个环境步抽样读一次；暂停时训练阻塞在 `_on_step` 的轮询里， -进程与显存保留、不消耗算力，点「继续」从同一个 step 边界恢复。暂停时长不计入 -`--max-wall-s`（否则"挂起一小时回来直接判超时"）。 - -「停止」同样是控制文件请求（**跨平台**）。`train_hrl_high.py` 有多层兜底， -点一下「停止」不再等于这一局白跑；而**急停**（强杀，例如 `SIGKILL` / -`taskkill /F`，进程没有任何收尾机会）也保证可恢复： - -| 机制 | 行为 | -| --- | --- | -| 滚动恢复点 | `--recovery-every`（默认 2000 环境步）把 `/_latest.zip` **原子覆盖写**一次；训练开始前先写一份**起步基线**。因此急停后最多只丢 `--recovery-every` 个环境步，且磁盘上始终有一份完整、可加载、可 `--resume` 的产物 | -| 原子落盘 | 模型 zip、旁挂 json、`resume.json` 都是「同目录临时文件 + `os.replace`」：保存途中被强杀不会留下半截文件，也不会破坏上一份完整快照 | -| 恢复指针 | `/resume.json` 记录当前最佳恢复目标 + 步数（增量语义）+ 生效过的 PPO 超参；前端顶部的「用最近检查点续训」按钮直接读它 | -| 定期 checkpoint | `--checkpoint-every`（默认 20000 环境步）另留 `_<步数>.zip`，`--checkpoint-keep` 只保留最近 K 个（默认 3），供返祖对比与多档恢复；滚动恢复点与起步基线不受清理影响 | -| 优雅停止 | 控制文件 `stop=true`（或 POSIX 信号 / Windows `SIGBREAK`）后，主进程在 step 边界收尾，保存最终模型与元数据（`stopped_by_signal=true`、`stop_signal`、`checkpoint_files`、`recovery_model`）并正常关闭环境 | -| 超时强杀 | 宽限期（`STOP_GRACE_S`，默认 15s）内没退出则强杀整棵进程树，快照 `forced_kill=true`，并在 `/forced_stop.json` 写明 `recovery.model`（可直接续训的目标） | -| 异常兜底 | `learn()` 抛出任何异常时先落 `*_emergency.zip` + 元数据 + `resume.json`，再写 `training_failure.json` 并抛出 | - -急停后恢复的操作方式：界面点「用最近检查点续训」；命令行则 -`--resume --model `，`--timesteps` 按**追加**步数理解。 - -checkpoint 与最终模型共用同一份 json 契约，因此 `--resume ` 直接 -续训（`--timesteps` 为增量步数）。注意 `--resume`/`--init` 会沿用 checkpoint 内的 -PPO 超参（`n_steps`/`batch_size`/`n_epochs`）：命令行这三项被忽略，脚本会打一条 -`hyperparams_from_checkpoint` 事件说明差异，元数据 `ppo_hyperparams` 记录**实际生效**值。 -回归测试见 `tests/test_hrl_checkpoint.py`，其中端到端用例真起 subproc 子进程，覆盖 -控制文件停止、POSIX 信号停止、**强杀后仍可续训**与暂停→继续（信号路径在 Windows -上自动跳过）。 - -## 测试 - -```bash -.venv/bin/python -m pytest tests/test_training_ui.py tests/test_training_ui_js.py tests/test_hrl_checkpoint.py -q -``` - -- `tests/test_training_ui.py`：配置契约同源、预设可直接提交、参数校验、SB3 表格 - 刻度打点、游标分页、`describe_process` 进度/ETA、示例快照与真实快照同键； - 用**桩训练脚本真起子进程**覆盖启动/轮询/暂停/继续/停止/退出码全链路；并覆盖 - 跨平台进程控制——控制文件往返与损坏容错、`spawn_kwargs` 平台分支、Windows 走 - `taskkill`、**无 `killpg` 环境下强杀不抛异常**（旧 bug 的直接回归）、控制文件被 - 忽略时强杀并写 `forced_stop.json` 恢复指引、`discover_recovery` 的 - `resume.json`/扫描/悬空指针三分支、HTTP `pause/resume`。 -- `tests/js/training_ui_harness.cjs`（经 `tests/test_training_ui_js.py` 调起）： - DOM 桩加载发布的 `charts.js`/`app.js` 原文，用服务端生成的真实快照驱动渲染， - 断言表单/指标卡/曲线/日志/逐回合表都拿到数据，覆盖增量游标三条分支与 - `classList`、HTML 转义、极端数值等边界。 -- `tests/test_hrl_checkpoint.py`：落盘与恢复的端到端回归——原子保存失败不破坏 - 上一份快照、JSON 原子替换、历史清理不误删滚动恢复点、起步基线可加载、 - **强杀急停后 `resume.json` 与滚动恢复点仍可继续训练**、控制文件停止落盘、 - 暂停→继续、POSIX 信号停止。 - -## 已知取舍 - -- 进度点保留上限 5000（服务端 deque 与前端本地缓存同值），超长训练只保留最近一段。 -- 表格解析依赖 SB3 的段名（`rollout/`/`time/`/`train/`/`system/`）；SB3 若改表格 - 格式，退化路径是「只显示 progress JSON」，不会导致界面崩溃。 -- 收敛判读是启发式；正式结论仍需用 `scripts/benchmark_hrl.py` 的配对评测与 - 学习曲线人工复核。 -- 暂停粒度是 `poll_every` 个环境步（默认 32），不是即时挂起；停止宽限期默认 15s。 -- Windows 强杀依赖系统自带的 `taskkill`；若被策略禁用会退回 `Popen.kill()` - （只杀主进程）。更严格的 Job Object 方案需要额外依赖，未引入。 - -Windows 兼容性问题的完整根因与修复记录见 -[`training-ui-windows-audit.md`](training-ui-windows-audit.md)。 diff --git a/docs/强化学习适用性审核与改进-20260911.md b/docs/强化学习适用性审核与改进-20260911.md deleted file mode 100644 index 1398258..0000000 --- a/docs/强化学习适用性审核与改进-20260911.md +++ /dev/null @@ -1,165 +0,0 @@ -# 第三、四问强化学习适用性审核与改进 - -本轮审核第一版强化学习代码、专家数据、模型及官方演练归档，修复训练与部署不一致的问题，训练两个新的 Maskable PPO 模型，并用独立实现的本地 `referee` 完成同场景对照。所有新性能数字均为本地结果，没有启动新的官方演练。 - -结论：强化学习适合在现有几何算法内尝试“下一检测点”的排序，第四问比第三问更有收益空间。当前证据支持第四问的混合方案优于手写 `directional` 基线；尚不能证明新模型稳定优于旧 PPO。第三问没有稳定增益，应保留 `bayes` 默认策略。新模型作为实验版本显式加载，没有覆盖旧默认模型。 - -## 1. 题设和方法适用范围 - -附录2(3)的交会必须使用带误差的角域，不能把两条示向线的交点当作真值；同地重测不会提供新的独立误差样本。附录2(4)的单接收机逐频道检测，结合动作规则，须计入移动、5秒检测及实际1秒换频。第三问的全向假设允许用无信号排除接收圆盘；第四问的无信号还可能来自方向遮挡，不能沿用这一排除结论。题设及几何算法的完整审核见[算法审核与优化](算法审核与优化-20260911.md)。 - -当前 RL 实际只替换 `RLBayesLoop._select_point` 和 `RLDirectionalLoop._next_probe`。七站/49站覆盖、频道发现、全局任务调度、保守候选集、清除证书和第四问后备清除仍由原算法负责。因此，完整场景全清验证的是混合系统，不是神经网络独立获得了完备性证明。 - -401维输入概括几何、历史读数和60个候选点，但省略完整历史、候选多边形细节及位置—朝向联合约束，不能直接称为完整马尔可夫状态。局部单源训练也没有学习全场覆盖路线与多源调度。PPO可以作为这种压缩观测下的近似策略尝试，但其目标并不自动等于全场最短时间；评估必须使用独立完整场景。[SB3官方训练与评估建议](https://stable-baselines3.readthedocs.io/en/master/guide/rl_tips.html)也强调独立评估、多随机种子和状态信息完整性。 - -## 2. 确认的实现缺陷及修复 - -| 缺陷 | 证据和影响 | 已实施修复 | -| --- | --- | --- | -| 训练传入整个 `BeliefState`，部署传入 `ChannelBelief` | 同一状态的特征索引7、14、15、17、18不同；训练丢失测量次数、末次结果和方位信息 | `env._observe` 使用目标频道信念；测试逐位比较部署输入，并覆盖跨频道交接 | -| BC使用ReLU，接收权重的PPO使用Tanh | 仅复制线性权重无法保留函数。原始专家输入探针中，Q3的90.96%、Q4的67.42%出现不同argmax | BC改为Tanh；复制时检查完整前向输出，重新初始化critic；旧ReLU/旧特征检查点拒绝直接迁移 | -| BC按观测行随机拆分 | 同一episode的相关状态可能同时出现在训练和验证中，分类准确率不足以说明独立场景泛化 | 保存episode编号，按episode划分训练/验证 | -| 第四问训练遮挡状态与运行时不一致 | 训练只记录距离保证盘内的无信号，且获得新方向后未正确重置连续遮挡；后备清除触发轨迹偏离部署 | 所有连续无信号记录为探测序列，仅盘内无信号用于确认定向；正观测清空序列，统一4次遮挡/8次定位阈值 | -| 奖励鼓励首次遮挡等中间事件 | “获得遮挡证据”额外+2、半径收缩等奖励会改变实际时间目标；较高回报不一定更快 | 主要奖励改为实际虚拟耗时的负值，成功清除统一+20，失败/截断重罚；取消这些中间奖励，采用gamma=1 | -| 未屏蔽同地重测、重合候选和第三问盘外梯子点 | 浪费固定误差条件下的检测时间；旧文档“所有候选保证接收”不成立 | 训练/推理共用动作屏蔽，排除已测0.5m格位置、重复候选、越界点；第三问存在距离保证盘时排除盘外点 | -| 保证盘半径恰为0时动作数量可能改变 | 依赖去重后的候选生成器会破坏固定60动作接口 | 即使位置重合也保留编号，交由动作屏蔽处理 | -| 只校验401/60维度不能识别语义变化 | 接收机频道被目标频道替代；旧模型与新模型同形但输入含义不同 | 加入特征版本与算法元数据，v2使用实际接收机频道，旧版保留原输入语义；拒绝问题编号或屏蔽模型元数据不匹配 | -| 基准将 `completed` 当作全清 | 内部状态自洽不代表全部真实源已清除 | 用评测端真实频道集合对照成功清除集合，并检查真实频道没有被误判缺席、正常退出、异常及逐步计时；旧基准新增 `all_valid` 和失败退出码 | -| 官方审计没有严格对应问题与策略名称 | 第三问/第四问摘要可能交叉通过名称检查 | `verify_rl_practice.py` 分别要求 `rl` / `rl_q4` | - -BC探针是对旧网络和“拷贝后换激活”的网络进行比较，并不是新模型错误率，也不是重新测得的BC验证准确率。旧报告38.9%/65.3%的分类准确率不能直接解释为已迁移PPO的模仿准确率。本轮训练的新模型均从随机初始化开始，未使用BC；BC修复通过前向一致性测试，未据此声称重新训练BC已经带来收益。 - -动作屏蔽采用官方 `MaskablePPO` 的 `action_masks` 接口，训练和确定性推理都传入相同规则。[SB3-Contrib官方文档](https://sb3-contrib.readthedocs.io/en/master/modules/ppo_mask.html)说明了这两处的用法。第四问仍允许探索盘外候选，距离保证不等于方向可见。退化到所有动作均被屏蔽时保留中心动作，由步数/预算边界处理，因此不能把本次零重复观测扩展为所有可能状态的严格保证。 - -正常成功episode的累计奖励为 `20 - 局部实际虚拟时间/100`；环境仍保留异常与失败处理，包括证书清除失败的额外惩罚。局部时间从首测与可选交接完成后开始，不能用训练报告中的局部均值替代整场成绩。第三问训练自动清除仍使用证书圆心，而完整 `BayesLoop` 可在证书范围内平移清除点；全局调度、预算与局部环境也有差异，完整闭环验证因此不可省略。 - -## 3. 训练和数据 - -第一版Q3为120000步随机初始化PPO；第一版Q4为BC初始化后40000步PPO。原专家数据分别为5141、14181条，原5场完整测试均值分别是Q3：3088.33/3124.76秒、Q4：14250.15/13230.20秒（手写/RL）。这些历史数字受旧环境和特征影响，不与本轮新课程的单源成功率直接比较。 - -修订课程保留首次有效测向，并加入35%的跨频道、跨位置交接，20%的区域边缘且最小接收半径场景。定向首测覆盖朝向±89.9°，更接近半平面边界。源位置、朝向、半径和逐点误差的抽样仍是自建分布，不是已知的官方生成分布；真值仅用于布设和评测，不进入策略观测或动作屏蔽。 - -两模型使用 `[128,128]` Tanh MLP、2个内存环境、`n_steps=512`、`batch_size=128`、6轮更新、学习率0.0003。实际均完成60416步，来自完整rollout批次向上取整；元数据中的60000是请求步数。安装版本为SB3 2.9.0、SB3-Contrib 2.9.0、Torch 2.11.0+cu128。本次选择CPU、单Torch线程，机器CUDA可用。 - -| 项目 | Q3新模型 | Q4新模型 | -| --- | ---: | ---: | -| 训练种子 | 310001 | 410001 | -| 定向概率 | 0 | 0.75 | -| 训练墙钟时间 | 276.6s | 250.6s | -| 独立单源评估成功 | 59/60 | 60/60 | -| 单源平均局部时间 | 270.04s | 349.51s | -| 平均动作步数 | 2.42 | 3.13 | - -Q3有一次局部训练任务未完成，不能声称单源成功率100%。完整闭环仍有自身调度与后备路径。两问题各只有一个新模型训练种子，未覆盖训练随机性。 - -## 4. 完整场景同条件对照 - -比较三种固定策略：当前手写基线（Q3 `bayes`、Q4 `directional`）、旧默认PPO、新Maskable PPO。相同场景使用相同源位置、频道、半径、朝向及逐点固定误差种子。`referee` 为单独实现，策略不读取模拟器真值。每场保存场景、完整请求/响应、summary、模型哈希和计时核验。 - -开发集为种子700—702、10/16源，各问题6场。随后固定新模型，不再据验证成绩调参；验证集为种子91000—91005、10/13/16源，各问题18场。第四问定向数覆盖1、约半数、仅一个全向三种组成。另加两种圆周边界压力场景。开发、验证、压力共52个不同场景，三策略共156次完整运行，全部满足真实源全清、缺席无误判、正常exit、零异常、逐步耗时复算一致。 - -### 开发集平均整场时间 - -| 问题 | 手写基线 | 旧PPO | 新模型 | -| --- | ---: | ---: | ---: | -| Q3，6场 | 3294.62s | 3206.20s | 3264.62s | -| Q4，6场 | 14289.76s | 12443.43s | 11758.72s | - -### 独立验证集 - -| 问题/指标 | 手写基线 | 旧PPO | 新模型 | -| --- | ---: | ---: | ---: | -| Q3有效全清 | 18/18 | 18/18 | 18/18 | -| Q3平均时间 | 3330.72s | 3254.61s | 3307.65s | -| Q3中位时间 | 3309.16s | 3262.64s | 3222.50s | -| Q3最大时间 | 3690.11s | 3786.98s | 3776.55s | -| Q4有效全清 | 18/18 | 18/18 | 18/18 | -| Q4平均时间 | 13778.82s | 12070.53s | 11797.57s | -| Q4中位时间 | 14587.80s | 12891.83s | 12488.23s | -| Q4最大时间 | 17273.98s | 15365.76s | 15894.44s | -| Q4累计未成功clear尝试 | 616 | 719 | 1380 | -| Q4累计同频道同点重测 | 12 | 28 | 0 | - -Q3三组都没有失败clear或同点重测。Q4新模型减少了重测，但更多使用后备光学清除尝试，平均耗时降低的同时尾部表现有所退化。失败clear本身可来自允许的后备搜索，不能仅凭该计数认定证书错误；其移动和3秒动作耗时全部纳入结果。 - -对独立验证场景做10000次配对bootstrap，负值表示新模型更快： - -| 对比 | 均值相对变化 | 95%场景重采样区间 | 新模型更快场数 | -| --- | ---: | ---: | ---: | -| Q3新模型对手写 | -0.69% | [-4.50%, +2.95%] | 9/18 | -| Q3新模型对旧PPO | +1.63% | [-1.64%, +5.19%] | 8/18 | -| Q4新模型对手写 | -14.38% | [-17.93%, -11.48%] | 18/18 | -| Q4新模型对旧PPO | -2.26% | [-6.08%, +1.71%] | 13/18 | - -新旧PPO的区间均跨0，不能据小幅平均变化宣称稳定改进。区间只反映固定模型在这些场景上的差异，不覆盖训练种子、未知官方分布或所有合法边界。此外本轮同时修改多项训练因素，这不是能归因到单个修复的消融实验。 - -### 边界压力与4000秒目标 - -源全部放在半径1800m圆周、接收半径1000m；第四问隔一个源采用向外发射方向。10/16源各一场，各策略都完成全清。 - -| 问题 | 手写平均 | 旧PPO平均 | 新模型平均 | -| --- | ---: | ---: | ---: | -| Q3 | 4472.58s | 4429.04s | 4648.26s | -| Q4 | 18248.87s | 14404.01s | 15210.08s | - -新Q3的两场分别为4659.06s、4637.46s，仍超过4000s；新Q4为14573.62s、15846.54s，且比旧PPO退化。此前几何优化在另一独立Q3验证组达到平均3315.89s、最大3981.74s，本轮新Q3验证组最大3776.55s；二者只能说明各自验证组达标，不能承诺全部题设场景小于4000s。第四问尚未达到4000s。 - -## 5. 官方证据与测试 - -重新审计已有官方归档，三次均通过题号/策略、官方包SHA、源数、完整清除、正常退出、请求时窗及逐步计时检查，未改写原归档： - -| 问题 | 既有官方案例 | 全清 | 整场时间 | -| --- | --- | ---: | ---: | -| Q3 | F38Y-9Q9J-B82V-CB9Z | 13/13 | 3538.81s | -| Q4，3个定向源 | X43X-VMRF-8GRZ-3SYA | 10/10 | 14003.09s | -| Q4，10个定向源 | BXSA-TFQ3-2DJ6-DHH7 | 10/10 | 14562.82s | - -这些是旧模型的官方可运行证据，不是新模型成绩；不同案例也不能直接计算优化百分比。新模型目前通过本地验证，未在官方新演练中运行。 - -全量回归：`python -m pytest -q --tb=short`，243项通过。初次沙箱运行有24个临时目录权限导致的fixture错误，获得权限后重跑全部通过。新增5项测试覆盖训练/部署观测、动作屏蔽、方向读数重置遮挡、实际成本奖励和BC完整前向一致性。之后对基准/官方审计入口的小修补分别用4次完整场景冒烟运行及三份官方归档复核通过。 - -## 6. 产物与复现 - -主目录为 `output/rl/review-20260911/`： - -- `q3_masked.zip` / `.json`、`q4_masked.zip` / `.json`：新模型与训练元数据，必须成对保存。 -- `evaluation/development/{controls,revised}/results.json`：开发对照。 -- `evaluation/validation/final/results.json`：独立验证与每场完整日志。 -- `evaluation/stress/final/results.json`：边界压力结果与日志。 -- `review_evidence.json`：配对区间、BC/特征错位探针、旧官方重审结果和代码哈希。 -- `benchmark-smoke.json`：旧基准有效性口径修复后的冒烟结果。 - -新Q3模型SHA-256：`b8b3a2bb103588348d8d7791bde9a2b0635c04778f0ac64f91c6f5c7cf87ca6e`。 -新Q4模型SHA-256：`6865d541ff9bdee81886d327345f2660d5f7d72f4b39810a58ae4b867ab37e42`。 - -从项目根目录复现训练时，使用新目录保留本轮检查点： - -```powershell -.venv/Scripts/python.exe scripts/train_rl.py --problem 3 --algorithm maskable_ppo --timesteps 60000 --seed 310001 --device cpu --model output/rl/reproduction/q3_masked.zip --eval-episodes 60 --verbose 0 -.venv/Scripts/python.exe scripts/train_rl.py --problem 4 --algorithm maskable_ppo --timesteps 60000 --seed 410001 --directional-probability 0.75 --device cpu --model output/rl/reproduction/q4_masked.zip --eval-episodes 60 --verbose 0 -``` - -复核本轮冻结模型的完整对照（label须使用尚不存在的名称）： - -```powershell -.venv/Scripts/python.exe scripts/evaluate_rl_revision.py --split validation --label rerun-validation -.venv/Scripts/python.exe scripts/evaluate_rl_revision.py --split stress --label rerun-stress -.venv/Scripts/python.exe scripts/summarize_rl_revision.py -``` - -本地内存自检可显式加载对应检查点；`--self-test`不会连接官方模拟器： - -```powershell -.venv/Scripts/mcm26b.exe run --robot-id 2026001 --self-test --strategy rl --rl-model output/rl/review-20260911/q3_masked.zip -.venv/Scripts/mcm26b.exe run --robot-id 2026001 --self-test --problem 4 --q4-strategy rl --rl-model output/rl/review-20260911/q4_masked.zip -``` - -在用户另行启动相应问题的官方演练后，使用实际登录队号和`--base-url`连接。例如第三问（路径使用本轮尚未创建的官方演练目录）： - -```powershell -.venv/Scripts/mcm26b.exe run --robot-id 2026001 --base-url http://127.0.0.1:2026 --problem 3 --strategy rl --rl-model output/rl/review-20260911/q3_masked.zip --log output/rl/review-20260911/practice-q3/session.jsonl -``` - -该命令保存请求/响应日志，汇总输出到终端；官方结果文件与汇总还需按原归档流程保存。第四问改用`--problem 4 --q4-strategy rl`及对应模型。本轮未执行上述官方连接命令。 - -进一步研究应先缩小单源训练与完整调度的差异，并处理Q4后备清除和边界退化，再做多个训练种子及新的冻结验证集。当前结果不足以将新模型自动替换为默认策略，也不足以把训练步数增加当作达到4000秒的依据。 diff --git a/logs/README.md b/logs/README.md deleted file mode 100644 index 9e2ea93..0000000 --- a/logs/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# logs/ - -正式测试/演练导出的 JSONL 与汇总日志放置目录。本地 P1 训练/评测日志实际写入 -`output/hrl/...`；本目录用于最终支撑材料归档，不修改文件名。 diff --git a/models/README.md b/models/README.md deleted file mode 100644 index 11bc1a3..0000000 --- a/models/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# models/ - -最终冻结模型归档目录。训练脚本默认把模型写到 `--model` 指定路径； -P1 smoke 产物见 `output/hrl/p1-smoke-time/`。正式提交前把冻结模型复制到本目录并保留同名 `.json` 元数据。 diff --git a/project/README.md b/project/README.md deleted file mode 100644 index 621437f..0000000 --- a/project/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# project/ 薄兼容层 - -题目 Prompt 要求 `project/` 目录树；本仓库的真实算法保持唯一一份， -位于 `src/mcm26b`、`scripts/`。本目录只做 re-export/转发，不复制实现。 - -| Prompt 路径 | 真实实现 | -|---|---| -| `env/simulator_client.py` | `src/mcm26b/protocol/client.py` | -| `env/local_env.py` | `src/mcm26b/hrl/hierarchical_env.py`, `low_level_env.py` | -| `env/particle_filter.py` | `src/mcm26b/particles.py` | -| `policies/high_level_ppo.py` | `src/mcm26b/hrl/policy.py` | -| `policies/low_level_ppo.py` | `src/mcm26b/hrl/policy.py`, `low_level_env.py` | -| `train/curriculum.py` | `src/mcm26b/hrl/curriculum.py` | -| `train/domain_randomization.py` | `src/mcm26b/hrl/curriculum.py` + 配置包装 | -| `eval/evaluate.py` | `scripts/benchmark_hrl.py` | -| `eval/ablation.py` | `scripts/ablate_hrl.py`（P2） | -| `baselines/heuristic.py` | `src/mcm26b/planner/hrl_loop.py` 启发式降级 | -| `baselines/end_to_end_ppo.py` | 显式未实现；单源局部 PPO 不冒充 E2E | - -使用示例： - -```python -from project.env.local_env import make_local_env -env = make_local_env(problem=3, stage=1, seed=7) -obs, info = env.reset(seed=7) -``` - -P1 当前交付：`benchmark_hrl.py`、`train_hrl_high.py`、CLI `--strategy hrl`、 -`--q4-strategy hrl`。P2 的 BC/Optuna/消融在 smoke 通过后另开。 diff --git a/project/__init__.py b/project/__init__.py deleted file mode 100644 index ac2fbef..0000000 --- a/project/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -"""题目 Prompt 所需目录树的薄兼容层。 - -真实实现仍位于 ``src/mcm26b``；本包只做 re-export / CLI 转发，避免两份算法逻辑。 -""" diff --git a/project/baselines/__init__.py b/project/baselines/__init__.py deleted file mode 100644 index 93ed785..0000000 --- a/project/baselines/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .heuristic import HeuristicHRL, run_heuristic - -__all__ = ["HeuristicHRL", "run_heuristic"] diff --git a/project/baselines/end_to_end_ppo.py b/project/baselines/end_to_end_ppo.py deleted file mode 100644 index e151f91..0000000 --- a/project/baselines/end_to_end_ppo.py +++ /dev/null @@ -1,16 +0,0 @@ -"""端到端 PPO 基线占位。 - -按本轮确认：当前只交付局部 PPO 基线；raw-observation/raw-action 的真正 -端到端 PPO 不在 P0+P1 范围内，因此显式标记未实现，避免把单源局部选点 -PPO 误称为端到端 baseline。 -""" -from __future__ import annotations - - -def train_end_to_end_ppo(*args, **kwargs): - raise NotImplementedError( - "本轮只交付局部 PPO 基线；真正的端到端 PPO 基线标记为未实现，" - "需单独立项定义原始观测、动作与安全执行边界。") - - -__all__ = ["train_end_to_end_ppo"] diff --git a/project/baselines/heuristic.py b/project/baselines/heuristic.py deleted file mode 100644 index 1034303..0000000 --- a/project/baselines/heuristic.py +++ /dev/null @@ -1,16 +0,0 @@ -"""兼容入口：信息增益/TSP/覆盖规则 + 高层启发式调度。""" -from mcm26b.planner.bayes_loop import BayesLoop -from mcm26b.planner.hrl_loop import HRLLoop - - -def HeuristicHRL(*args, **kwargs): # pylint: disable=invalid-name # 兼容入口保留题面公开名 - """返回无模型的高层 HRL 闭环（heuristic_action 降级路径）。""" - kwargs.setdefault("high_policy", None) - return HRLLoop(*args, **kwargs) - - -def run_heuristic(*args, **kwargs): - return HeuristicHRL(*args, **kwargs).run() - - -__all__ = ["HeuristicHRL", "run_heuristic", "BayesLoop", "HRLLoop"] diff --git a/project/env/__init__.py b/project/env/__init__.py deleted file mode 100644 index 502254f..0000000 --- a/project/env/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from .simulator_client import SimulatorClient -from .local_env import HierarchicalEnv, LowLevelEnv, HRLReward, make_local_env -from .particle_filter import ChannelParticles - -__all__ = ["SimulatorClient", "HierarchicalEnv", "LowLevelEnv", "HRLReward", - "make_local_env", "ChannelParticles"] diff --git a/project/env/local_env.py b/project/env/local_env.py deleted file mode 100644 index d5ff04b..0000000 --- a/project/env/local_env.py +++ /dev/null @@ -1,22 +0,0 @@ -"""兼容入口：本地简化训练环境。 - -HierarchicalEnv 是高层宏动作环境；LowLevelEnv 是独立八方向导航环境。 -本文件不复制实现，只转发 mcm26b.hrl 下的真实类。 -""" -from mcm26b.hrl.hierarchical_env import HRLReward, HierarchicalEnv -from mcm26b.hrl.low_level_env import LowLevelEnv - -__all__ = ["HierarchicalEnv", "LowLevelEnv", "HRLReward", "make_local_env"] - - -def make_local_env(*, problem: int = 3, stage: int = 4, seed: int | None = None, - fidelity: str = "production"): - """按题目 Prompt 的四阶段编号创建训练环境。""" - from mcm26b.hrl.curriculum import STAGES - if not 1 <= int(stage) <= len(STAGES): - raise ValueError(f"stage 必须在 1..{len(STAGES)}") - env = HierarchicalEnv(problem=problem, stage=STAGES[int(stage) - 1], - fidelity=fidelity) - if seed is not None: - env.reset(seed=seed) - return env diff --git a/project/env/particle_filter.py b/project/env/particle_filter.py deleted file mode 100644 index f9c6d9e..0000000 --- a/project/env/particle_filter.py +++ /dev/null @@ -1,4 +0,0 @@ -"""兼容入口：逐频道网格粒子/幸存集。""" -from mcm26b.particles import ChannelParticles, angular_gap - -__all__ = ["ChannelParticles", "angular_gap"] diff --git a/project/env/simulator_client.py b/project/env/simulator_client.py deleted file mode 100644 index 43c1e56..0000000 --- a/project/env/simulator_client.py +++ /dev/null @@ -1,8 +0,0 @@ -"""兼容入口：真实类见 mcm26b.protocol.client。""" -from mcm26b.protocol.client import (ActionRejected, ClearOutcome, DeadlineExceeded, - ExitInfo, MeasureOutcome, ProtocolError, - ProtocolViolation, SimulatorClient) - -__all__ = ["SimulatorClient", "MeasureOutcome", "ClearOutcome", "ExitInfo", - "ProtocolError", "ActionRejected", "ProtocolViolation", - "DeadlineExceeded"] diff --git a/project/eval/__init__.py b/project/eval/__init__.py deleted file mode 100644 index 1fd6c71..0000000 --- a/project/eval/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .evaluate import run_benchmark -from .ablation import run_ablation - -__all__ = ["run_benchmark", "run_ablation"] diff --git a/project/eval/ablation.py b/project/eval/ablation.py deleted file mode 100644 index a4696ce..0000000 --- a/project/eval/ablation.py +++ /dev/null @@ -1,20 +0,0 @@ -"""兼容入口：调用 scripts/ablate_hrl.py（P2 后启用）。""" -from __future__ import annotations - -import importlib.util -from pathlib import Path - - -def run_ablation(argv=None) -> int: - path = Path(__file__).resolve().parents[2] / "scripts" / "ablate_hrl.py" - if not path.exists(): - raise NotImplementedError( - "scripts/ablate_hrl.py 尚未实现；P1 阶段只保证 benchmark/training/CLI。") - spec = importlib.util.spec_from_file_location("ablate_hrl_compat", path) - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) - return int(module.main(argv)) - - -__all__ = ["run_ablation"] diff --git a/project/eval/evaluate.py b/project/eval/evaluate.py deleted file mode 100644 index 8eb7e21..0000000 --- a/project/eval/evaluate.py +++ /dev/null @@ -1,17 +0,0 @@ -"""兼容入口：调用 scripts/benchmark_hrl.py。""" -from __future__ import annotations - -import importlib.util -from pathlib import Path - - -def run_benchmark(argv=None) -> int: - path = Path(__file__).resolve().parents[2] / "scripts" / "benchmark_hrl.py" - spec = importlib.util.spec_from_file_location("benchmark_hrl_compat", path) - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) - return int(module.main(argv)) - - -__all__ = ["run_benchmark"] diff --git a/project/policies/__init__.py b/project/policies/__init__.py deleted file mode 100644 index a52064f..0000000 --- a/project/policies/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .high_level_ppo import HighLevelPolicy, load_high_policy -from .low_level_ppo import LowLevelPolicy, load_low_policy - -__all__ = ["HighLevelPolicy", "load_high_policy", - "LowLevelPolicy", "load_low_policy"] diff --git a/project/policies/high_level_ppo.py b/project/policies/high_level_ppo.py deleted file mode 100644 index 4dd4407..0000000 --- a/project/policies/high_level_ppo.py +++ /dev/null @@ -1,10 +0,0 @@ -"""兼容入口：高层 MaskablePPO 推理策略。""" -from mcm26b.hrl.policy import HighLevelPolicy - - -def load_high_policy(model_path: str, *, device: str = "auto", - problem: int | None = None) -> HighLevelPolicy: - return HighLevelPolicy(model_path, device=device, problem=problem) - - -__all__ = ["HighLevelPolicy", "load_high_policy"] diff --git a/project/policies/low_level_ppo.py b/project/policies/low_level_ppo.py deleted file mode 100644 index 974efb2..0000000 --- a/project/policies/low_level_ppo.py +++ /dev/null @@ -1,9 +0,0 @@ -"""兼容入口：低层八方向到达策略（离线导航实验）。""" -from mcm26b.hrl.policy import LowLevelPolicy - - -def load_low_policy(model_path: str, *, device: str = "auto") -> LowLevelPolicy: - return LowLevelPolicy(model_path, device=device) - - -__all__ = ["LowLevelPolicy", "load_low_policy"] diff --git a/project/train/__init__.py b/project/train/__init__.py deleted file mode 100644 index cf0ca9e..0000000 --- a/project/train/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .curriculum import STAGES, Stage, make_scene, training_config -from .domain_randomization import randomize_problem_config - -__all__ = ["STAGES", "Stage", "make_scene", "training_config", - "randomize_problem_config"] diff --git a/project/train/curriculum.py b/project/train/curriculum.py deleted file mode 100644 index b99220d..0000000 --- a/project/train/curriculum.py +++ /dev/null @@ -1,6 +0,0 @@ -"""兼容入口：课程学习与域随机化。""" -from mcm26b.hrl.curriculum import (STAGES, Stage, make_scene, stage_with_noise, - training_config, training_spec) - -__all__ = ["STAGES", "Stage", "make_scene", "stage_with_noise", - "training_config", "training_spec"] diff --git a/project/train/domain_randomization.py b/project/train/domain_randomization.py deleted file mode 100644 index d585db2..0000000 --- a/project/train/domain_randomization.py +++ /dev/null @@ -1,31 +0,0 @@ -"""兼容入口：本地域随机化参数包装。 - -当前真实随机化在 mcm26b.hrl.curriculum.make_scene 内完成；这里只暴露 -ProblemConfig 的显式替换助手，便于 Optuna/消融在实验配置中记录随机化范围。 -""" -from __future__ import annotations - -import dataclasses -from random import Random - -from mcm26b.config import ProblemConfig - - -def randomize_problem_config(base: ProblemConfig, *, seed: int, - bearing_error_half_deg: float | None = None, - r_eff_range: tuple[float, float] | None = None - ) -> ProblemConfig: - """返回带域随机化标量的新配置；不修改全局默认配置。""" - _rng = Random(seed) # 固定种子留作兼容：真实随机化在 make_scene 内完成 - field = base.field - if bearing_error_half_deg is None: - bearing_error_half_deg = field.bearing_error_half_deg - if r_eff_range is not None: - lo, hi = r_eff_range - if not 0 <= lo <= hi: - raise ValueError("r_eff_range 必须为递增非负区间") - return dataclasses.replace(base, field=dataclasses.replace( - field, bearing_error_half_deg=float(bearing_error_half_deg))) - - -__all__ = ["randomize_problem_config"] diff --git a/project/train/optuna_tune.py b/project/train/optuna_tune.py deleted file mode 100644 index b666668..0000000 --- a/project/train/optuna_tune.py +++ /dev/null @@ -1,20 +0,0 @@ -"""兼容入口：调用 scripts/tune_hrl.py（P2 后启用）。""" -from __future__ import annotations - -import importlib.util -from pathlib import Path - - -def run_tuning(argv=None) -> int: - path = Path(__file__).resolve().parents[2] / "scripts" / "tune_hrl.py" - if not path.exists(): - raise NotImplementedError( - "scripts/tune_hrl.py 尚未实现；P1 阶段先冻结超参并留待 P2。") - spec = importlib.util.spec_from_file_location("tune_hrl_compat", path) - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) - return int(module.main(argv)) - - -__all__ = ["run_tuning"] diff --git a/pyproject.toml b/pyproject.toml index 7555cd0..27188ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,30 +15,13 @@ dev = [ # 独立评测机（自建模拟器）：仅测试侧依赖，运行时代码零导入（懒加载 + 守卫测试） "mcm26b-referee", ] -rl = [ - "gymnasium>=1.1", - "sb3-contrib==2.9.0", - "stable-baselines3>=2.6", - "torch==2.11.0", -] # uv 工作区：referee/ 是独立包（零第三方依赖、不依赖 mcm26b），双方仅通过 HTTP 协议耦合 [tool.uv.workspace] members = ["referee"] -# RL 训练使用 CUDA 12.8 版 PyTorch（本机 RTX 4060；运行时依赖保持轻量）。 -[[tool.uv.index]] -name = "pytorch-cu128" -url = "https://download.pytorch.org/whl/cu128" -explicit = true - [tool.uv.sources] mcm26b-referee = { workspace = true } -# CUDA 12.8 版 PyTorch 仅限 Windows/Linux（RTX 4060 训练机）； -# macOS (Apple Silicon, MPS) 回落到 PyPI 官方轮子。 -torch = [ - { index = "pytorch-cu128", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, -] [build-system] requires = ["hatchling"] @@ -102,7 +85,7 @@ disable = [ "missing-module-docstring", # 内部小模块与 __main__ 不强制 "missing-class-docstring", "missing-function-docstring", - "import-outside-toplevel", # torch/SB3/referee 等重依赖与可选依赖懒加载 + "import-outside-toplevel", # referee 等重依赖与可选依赖懒加载 "wrong-import-position", # 同上：importorskip 守卫后的延迟导入 "ungrouped-imports", # 同上：延迟导入无法与顶部导入分组 "broad-exception-caught", # 子进程 / IO / HTTP 边界防御式兜底，逐处注释 diff --git a/scripts/ablate_hrl.py b/scripts/ablate_hrl.py deleted file mode 100644 index c038a62..0000000 --- a/scripts/ablate_hrl.py +++ /dev/null @@ -1,25 +0,0 @@ -"""HRL 消融入口（P2 待实现）。 - -预定义消融维度：时间目标 vs prompt 塑形、信息增益奖励系数、动作掩码取消范围、 -课程学习、粒子统计特征。硬性协议守卫/证书/预算永远保留，不在消融中取消。 -""" -from __future__ import annotations - -import argparse - - -def main(argv=None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--variants", default="time,prompt,no_info,no_mask,no_curriculum") - parser.add_argument("--manifest", default="output/hrl/manifests/final.json") - parser.add_argument("--output", default="output/hrl/p2-ablation") - parser.add_argument("--timesteps", type=int, default=2048) - parser.add_argument("--seeds", type=int, default=5) - parser.parse_args(argv) - raise NotImplementedError( - "P2 尚未实现：必须先完成专家 BC 与冻结场景对照，" - "再对相同训练步数/场景/种子逐项消融；P1 只保证 benchmark/training/CLI。") - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/benchmark_hrl.py b/scripts/benchmark_hrl.py deleted file mode 100644 index 443090d..0000000 --- a/scripts/benchmark_hrl.py +++ /dev/null @@ -1,428 +0,0 @@ -"""HRL/基线统一评测入口（P1）。 - -特点： -- 同一 manifest 中的真值场景被所有 variant 复用，保证配对比较； -- 真值仅在测试端用于构造 ``LocalSimulator`` 与最终核对，策略端只收到协议响应； -- 支持 ``hrl_heuristic``（高层启发式降级）、``bayes``、``directional``； -- 支持已训练高层模型 ``hrl``（需 ``--hrl-model`` 或 manifest 中的 variant 配置）。 - -输出目录（默认拒绝覆盖已有结果）： - /manifest.json - /runs///summary.json - /runs///session.jsonl - /results.json - /aggregate.json - /aggregate.csv - -示例： - python scripts/benchmark_hrl.py \ - --variants hrl_heuristic,bayes,directional \ - --manifest output/hrl/manifests/dev.json \ - --output output/hrl/p1-heuristic -""" - -from __future__ import annotations - -import argparse -import csv -import json -import statistics -import sys -import time -from collections import defaultdict -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -from mcm26b.config import ProblemConfig -from mcm26b.harness import LocalSimTransport, mixed_scene, random_omni_scene -from mcm26b.harness.scenes import referee_spec -from mcm26b.protocol import SimulatorClient -from mcm26b.protocol.log import SessionLog - -try: # 测试端依赖；运行时代码不导入 - from referee import LocalSimulator, Source -except ImportError as exc: # pragma: no cover - raise SystemExit("benchmark_hrl 需要 dev 依赖 referee。请先 `uv sync`。") from exc - - -# --------------------------------------------------------------------------- -# manifest / scene -# --------------------------------------------------------------------------- -@dataclass(frozen=True) -class SceneSpec: - scene_id: str - seed: int - error_seed: int - problem: int - sources: list[dict[str, Any]] - - @property - def truth_channels(self) -> tuple[int, ...]: - return tuple(sorted(int(src["channel"]) for src in self.sources)) - - -def _source_dict(source: Source) -> dict[str, Any]: - return { - "channel": int(source.channel), - "x": float(source.x), - "y": float(source.y), - "r_eff_m": float(source.r_eff_m), - "directional": bool(source.directional), - "heading_deg": float(source.heading_deg), - } - - -def _sources_to_dicts(sources: list[Source]) -> list[dict[str, Any]]: - return [_source_dict(source) for source in sources] - - -def _source_from_dict(data: dict[str, Any]) -> Source: - return Source( - channel=int(data["channel"]), - x=float(data["x"]), - y=float(data["y"]), - r_eff_m=float(data["r_eff_m"]), - directional=bool(data.get("directional", False)), - heading_deg=float(data.get("heading_deg", 0.0)), - ) - - -def make_default_manifest(*, problem: int, scenes: int, sources: int | None, - directional: int, start_seed: int, - config: ProblemConfig) -> dict[str, Any]: - """按 CLI 参数生成可复现的固定场景清单。""" - entries: list[dict[str, Any]] = [] - for index in range(scenes): - seed = start_seed + index - if problem == 4: - scene_sources = mixed_scene(seed=seed, count=sources, - directional=directional, config=config) - else: - scene_sources = random_omni_scene(seed=seed, count=sources, - config=config) - entries.append({ - "scene_id": f"scene-{seed}", - "seed": seed, - "error_seed": seed + 10_000, - "problem": problem, - "sources": _sources_to_dicts(scene_sources), - }) - return {"version": 1, "problem": problem, "scenes": entries} - - -def load_or_generate_manifest(path: Path, *, problem: int, scenes: int, - sources: int | None, directional: int, - start_seed: int, - config: ProblemConfig) -> dict[str, Any]: - if path.exists(): - manifest = json.loads(path.read_text(encoding="utf-8")) - else: - manifest = make_default_manifest( - problem=problem, scenes=scenes, sources=sources, - directional=directional, start_seed=start_seed, config=config) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2), - encoding="utf-8") - if "scenes" not in manifest or not manifest["scenes"]: - raise ValueError(f"manifest 无 scenes: {path}") - for raw in manifest["scenes"]: - if not {"scene_id", "error_seed", "problem", "sources"} <= set(raw): - raise ValueError(f"manifest scene 字段缺失: {raw!r}") - if not raw["sources"]: - raise ValueError(f"scene {raw.get('scene_id')} 没有来源真值") - return manifest - - -def scenes_from_manifest(manifest: dict[str, Any]) -> list[SceneSpec]: - result: list[SceneSpec] = [] - for raw in manifest["scenes"]: - result.append(SceneSpec( - scene_id=str(raw["scene_id"]), - seed=int(raw.get("seed", 0)), - error_seed=int(raw["error_seed"]), - problem=int(raw["problem"]), - sources=[dict(item) for item in raw["sources"]], - )) - return result - - -# --------------------------------------------------------------------------- -# variants -# --------------------------------------------------------------------------- -def _build_loop(variant: str, client: SimulatorClient, config: ProblemConfig, - problem: int, *, hrl_model: str | None, - hrl_device: str = "auto"): - """延迟导入变体实现，避免 benchmark 顶层拉起重依赖。""" - if variant == "bayes": - from mcm26b.planner.bayes_loop import BayesLoop - return BayesLoop(client, config) - if variant == "directional": - from mcm26b.planner import DirectionalExtension - return DirectionalExtension(client, config) - if variant == "hrl_heuristic": - from mcm26b.planner.hrl_loop import HRLLoop - return HRLLoop(client, config, problem=problem, high_policy=None) - if variant == "hrl": - if not hrl_model: - raise RuntimeError("variant=hrl 需要 --hrl-model") - from mcm26b.hrl.policy import HighLevelPolicy - from mcm26b.planner.hrl_loop import HRLLoop - policy = HighLevelPolicy(hrl_model, device=hrl_device, problem=problem) - return HRLLoop(client, config, problem=problem, high_policy=policy) - raise ValueError(f"未知 variant: {variant!r}") - - -def _validity(summary: dict[str, Any], truth_channels: tuple[int, ...]) -> dict[str, Any]: - """测试端有效性审计。真值只在此处进入。""" - cleared = set(int(c) for c in summary.get("cleared_channels", [])) - absent = set(int(c) for c in summary.get("absent_channels", [])) - truth = set(truth_channels) - declared_absent = truth & absent - cleared_matches_truth = cleared == truth - no_source_declared_absent = not declared_absent - completed = bool(summary.get("completed")) - anomalies = list(summary.get("anomalies", [])) - # 本地 benchmark 要求：清洗真值且无守恒性异常；缺席集合不得含真值。 - valid = completed and cleared_matches_truth and no_source_declared_absent \ - and not anomalies - return { - "valid": valid, - "completed": completed, - "cleared_matches_truth": cleared_matches_truth, - "no_source_declared_absent": no_source_declared_absent, - "truth_channels": sorted(truth), - "declared_absent_truth": sorted(declared_absent), - "anomaly_count": len(anomalies), - } - - -def _row_from_summary(*, scene: SceneSpec, variant: str, summary: dict[str, Any], - wall_s: float) -> dict[str, Any]: - validity = _validity(summary, scene.truth_channels) - cleared_count = len(summary.get("cleared_channels", [])) - virtual_s = float(summary.get("virtual_time_s") or 0.0) - avg_s = summary.get("avg_locate_clear_time_s") - totals = summary.get("step_totals", {}) or {} - return { - "scene_id": scene.scene_id, - "variant": variant, - "problem": scene.problem, - "truth_count": len(scene.truth_channels), - "cleared_count": cleared_count, - "virtual_time_s": virtual_s, - "avg_locate_clear_time_s": (float(avg_s) if avg_s is not None else None), - "accounted_virtual_time_s": summary.get("accounted_virtual_time_s"), - "program_run_time_s": round(wall_s, 6), - "real_elapsed_total_s": totals.get("real_elapsed_total_s"), - "total_requests": summary.get("total_requests"), - "measures": summary.get("measures"), - "clears": summary.get("clears"), - "clear_success": summary.get("clear_success"), - "completed": bool(summary.get("completed")), - "stop_reason": summary.get("stop_reason"), - "exit_reason": summary.get("exit_reason"), - **validity, - } - - -def _error_row(scene: SceneSpec, variant: str, exc: BaseException) -> dict[str, Any]: - return { - "scene_id": scene.scene_id, - "variant": variant, - "problem": scene.problem, - "truth_count": len(scene.truth_channels), - "cleared_count": 0, - "virtual_time_s": None, - "avg_locate_clear_time_s": None, - "accounted_virtual_time_s": None, - "program_run_time_s": None, - "real_elapsed_total_s": None, - "total_requests": None, - "measures": None, - "clears": None, - "clear_success": None, - "completed": False, - "stop_reason": None, - "exit_reason": None, - "valid": False, - "cleared_matches_truth": False, - "no_source_declared_absent": False, - "truth_channels": sorted(scene.truth_channels), - "declared_absent_truth": [], - "anomaly_count": 0, - "error": f"{type(exc).__name__}: {exc}", - } - - -def run_scene_variant(scene: SceneSpec, variant: str, *, config: ProblemConfig, - hrl_model: str | None, run_dir: Path, - hrl_device: str = "auto") -> dict[str, Any]: - """运行一个 scene × variant。返回可序列化结果行与 summary。""" - run_dir.mkdir(parents=True, exist_ok=True) - sources = [_source_from_dict(raw) for raw in scene.sources] - simulator = LocalSimulator( - sources, - robot_id="hrl-benchmark", - spec=referee_spec(config), - error_seed=scene.error_seed, - stress_mode=False, - ) - # 真值只用于 LocalSimulator；策略端仅持有 client/loop，不读取 simulator.sources。 - log_path = run_dir / "session.jsonl" - log = SessionLog(path=log_path) - client = SimulatorClient("hrl-benchmark", config=config, - transport=LocalSimTransport(simulator), log=log) - loop = _build_loop(variant, client, config, scene.problem, - hrl_model=hrl_model, hrl_device=hrl_device) - started = time.monotonic() - try: - summary = loop.run() - finally: - wall_s = time.monotonic() - started - row = _row_from_summary(scene=scene, variant=variant, summary=summary, - wall_s=wall_s) - # 完整 summary 可能很大；仍写入 runs 目录，供复算与异常审计。 - (run_dir / "summary.json").write_text( - json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8") - return {"row": row, "summary": summary} - - -# --------------------------------------------------------------------------- -# aggregate -# --------------------------------------------------------------------------- -def _aggregate(rows: list[dict[str, Any]]) -> dict[str, Any]: - grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) - for row in rows: - grouped[row["variant"]].append(row) - result: dict[str, Any] = {"variants": {}, "all_valid": True} - for variant, items in sorted(grouped.items()): - virtual = [float(item["virtual_time_s"]) for item in items - if item.get("virtual_time_s") is not None] - per_source = [float(item["avg_locate_clear_time_s"]) - for item in items - if item.get("avg_locate_clear_time_s") is not None] - cleared = [int(item["cleared_count"] or 0) for item in items] - program = [float(item["program_run_time_s"]) for item in items - if item.get("program_run_time_s") is not None] - success = [1.0 if item.get("valid") else 0.0 for item in items] - result["variants"][variant] = { - "episodes": len(items), - "valid_episodes": int(sum(success)), - "valid_rate": (sum(success) / len(items)) if items else 0.0, - "success_rate": (sum(success) / len(items)) if items else 0.0, - "cleared_count_mean": statistics.fmean(cleared) if cleared else 0.0, - "cleared_count_std": (statistics.pstdev(cleared) - if len(cleared) > 1 else 0.0), - "virtual_time_mean_s": statistics.fmean(virtual) if virtual else 0.0, - "virtual_time_std_s": (statistics.pstdev(virtual) - if len(virtual) > 1 else 0.0), - "virtual_time_min_s": min(virtual) if virtual else 0.0, - "virtual_time_max_s": max(virtual) if virtual else 0.0, - "avg_locate_clear_time_mean_s": (statistics.fmean(per_source) - if per_source else None), - "program_run_time_mean_s": (statistics.fmean(program) - if program else None), - "total_requests_mean": statistics.fmean( - [int(item.get("total_requests") or 0) for item in items]), - } - if result["variants"][variant]["valid_rate"] < 1.0: - result["all_valid"] = False - return result - - -def main(argv=None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--variants", default="hrl_heuristic,bayes,directional", - help="逗号分隔：hrl_heuristic,bayes,directional,hrl") - parser.add_argument("--manifest", default="output/hrl/manifests/dev.json") - parser.add_argument("--output", default="output/hrl/p1-benchmark") - parser.add_argument("--problem", type=int, choices=(3, 4), default=3) - parser.add_argument("--scenes", type=int, default=6, - help="manifest 不存在时生成的场景数") - parser.add_argument("--sources", type=int, default=None, - help="每局源数；默认按题面 10~16 随机") - parser.add_argument("--directional", type=int, default=3, - help="问题4定向源数量（生成 manifest 时使用）") - parser.add_argument("--start-seed", type=int, default=91_000) - parser.add_argument("--hrl-model", default=None) - parser.add_argument("--hrl-device", choices=("auto", "cpu", "cuda", "mps"), - default="auto") - parser.add_argument("--force", action="store_true", - help="允许覆盖已有输出目录") - parser.add_argument("--log", action="store_true", - help="为兼容 CLI 保留；当前始终写 session.jsonl") - args = parser.parse_args(argv) - - config = ProblemConfig() - output = Path(args.output) - if output.exists() and any(output.iterdir()) and not args.force: - print(f"输出目录已存在且非空：{output}；如需覆盖请加 --force。", - file=sys.stderr) - return 2 - output.mkdir(parents=True, exist_ok=True) - manifest_path = Path(args.manifest) - manifest = load_or_generate_manifest( - manifest_path, problem=args.problem, scenes=args.scenes, - sources=args.sources, directional=args.directional, - start_seed=args.start_seed, config=config) - scenes = scenes_from_manifest(manifest) - variants = [item.strip() for item in args.variants.split(",") if item.strip()] - if not variants: - parser.error("--variants 不能为空") - if "hrl" in variants and not args.hrl_model: - # manifest 中也允许写明模型路径；此处先支持 CLI 显式路径。 - print("variant=hrl 需要 --hrl-model（P1 冒烟模型路径）。", file=sys.stderr) - return 2 - known = {"hrl_heuristic", "bayes", "directional", "hrl"} - unknown = sorted(set(variants) - known) - if unknown: - parser.error(f"未知 variant: {unknown}") - - rows: list[dict[str, Any]] = [] - runs_root = output / "runs" - wall_started = time.monotonic() - for scene in scenes: - for variant in variants: - run_dir = runs_root / scene.scene_id / variant - print(json.dumps({"event": "run_start", "scene": scene.scene_id, - "variant": variant}, ensure_ascii=False)) - try: - result = run_scene_variant( - scene, variant, config=config, hrl_model=args.hrl_model, - run_dir=run_dir, hrl_device=args.hrl_device) - row = result["row"] - except Exception as exc: # 单场异常也进入聚合，all_valid=false - row = _error_row(scene, variant, exc) - rows.append(row) - print(json.dumps({"event": "run_done", **{ - key: row.get(key) for key in - ("scene_id", "variant", "cleared_count", "virtual_time_s", - "valid", "program_run_time_s", "error")}}, ensure_ascii=False)) - aggregate = _aggregate(rows) - aggregate["wall_seconds"] = round(time.monotonic() - wall_started, 3) - aggregate["manifest"] = str(manifest_path) - aggregate["output"] = str(output) - aggregate["variants_requested"] = variants - aggregate["scenes"] = len(scenes) - (output / "results.json").write_text( - json.dumps(rows, ensure_ascii=False, indent=2), encoding="utf-8") - (output / "manifest.json").write_text( - json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8") - (output / "aggregate.json").write_text( - json.dumps(aggregate, ensure_ascii=False, indent=2), encoding="utf-8") - fieldnames = list(rows[0].keys()) - for row in rows: - for key in row: - if key not in fieldnames: - fieldnames.append(key) - with (output / "aggregate.csv").open("w", newline="", encoding="utf-8") as handle: - writer = csv.DictWriter(handle, fieldnames=fieldnames) - writer.writeheader() - writer.writerows(rows) - print(json.dumps(aggregate, ensure_ascii=False, indent=2)) - return 0 if aggregate.get("all_valid") else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/benchmark_rl.py b/scripts/benchmark_rl.py deleted file mode 100644 index fbfd055..0000000 --- a/scripts/benchmark_rl.py +++ /dev/null @@ -1,128 +0,0 @@ -"""问题3/4 RL 策略与既有策略的配对自建消融。 - -同一种子、同一场景分别跑基线（Q3 bayes / Q4 directional）与 RL 变体， -比较全部清除成功、虚拟总时间、检测次数与请求总数。结果写入 -output/rl/benchmark.json。 - -用法： - uv sync --group rl --group dev - uv run python scripts/benchmark_rl.py --seeds 5 -""" - -from __future__ import annotations - -import argparse -import json -import statistics -import time -from pathlib import Path - -from mcm26b.config import ProblemConfig -from mcm26b.evaluation import recompute_virtual_time -from mcm26b.harness import (LocalSimTransport, mixed_scene, random_omni_scene, - referee_spec) -from mcm26b.planner import DirectionalExtension -from mcm26b.planner.bayes_loop import BayesLoop -from mcm26b.planner.rl_loop import RLBayesLoop, RLDirectionalLoop -from mcm26b.protocol.client import SimulatorClient -from referee import LocalSimulator - - -def run_one(loop_cls, sources, *, config, robot_id: str, **kwargs) -> dict: - simulator = LocalSimulator(sources, robot_id=robot_id, - spec=referee_spec(config, max_real_duration_s=3600.0), - stress_mode=True) - client = SimulatorClient(robot_id, config=config, - transport=LocalSimTransport(simulator)) - started = time.monotonic() - summary = loop_cls(client, config, **kwargs).run() \ - if kwargs else loop_cls(client, config).run() - events = client.log.action_events() - successful = {e['request']['channel'] for e in events - if e['response'].get('clear_result') == 'success'} - truth = {s.channel for s in sources} - return { - "completed": bool(summary["completed"]), - "all_cleared": successful == truth, - "correct_absence": not bool(set(summary['absent_channels']) & truth), - "anomalies": summary['anomalies'], - "timing_mismatches": len(recompute_virtual_time(events)['mismatches']), - "normal_exit": summary['exit_reason'] == 'user_exit', - "virtual_time_s": float(summary["virtual_time_s"]), - "measures": int(summary["measures"]), - "total_requests": int(summary["total_requests"]), - "move_distance_m": float(summary["move_distance_m"]), - "wall_s": round(time.monotonic() - started, 3), - } - - -def main(argv=None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--seeds", type=int, default=5) - parser.add_argument("--sources", type=int, default=10) - parser.add_argument("--directional", type=int, default=3) - parser.add_argument("--output", default="output/rl/benchmark.json") - args = parser.parse_args(argv) - config = ProblemConfig() - robot_id = "2026001" - report = {"config": {"seeds": args.seeds, "sources": args.sources, - "directional": args.directional}, - "q3": [], "q4": []} - - for seed in range(20260910, 20260910 + args.seeds): - omni = random_omni_scene(seed=seed, count=args.sources, config=config) - mixed = mixed_scene(seed=seed, count=args.sources, - directional=args.directional, config=config) - q3_base = run_one(BayesLoop, omni, config=config, robot_id=robot_id) - q3_rl = run_one(RLBayesLoop, omni, config=config, robot_id=robot_id) - q4_base = run_one(DirectionalExtension, mixed, config=config, - robot_id=robot_id) - q4_rl = run_one(RLDirectionalLoop, mixed, config=config, - robot_id=robot_id) - report["q3"].append({"seed": seed, "bayes": q3_base, "rl": q3_rl}) - report["q4"].append({"seed": seed, "directional": q4_base, "rl": q4_rl}) - print(json.dumps({"event": "seed_done", "seed": seed, - "q3_base_s": q3_base["virtual_time_s"], - "q3_rl_s": q3_rl["virtual_time_s"], - "q4_base_s": q4_base["virtual_time_s"], - "q4_rl_s": q4_rl["virtual_time_s"]}, - ensure_ascii=False)) - - for key, baseline in (("q3", "bayes"), ("q4", "directional")): - rows = report[key] - deltas = [row["rl"]["virtual_time_s"] - row[baseline]["virtual_time_s"] - for row in rows] - cleared = all(row["rl"]["all_cleared"] and row[baseline]["all_cleared"] - for row in rows) - valid = all(part['completed'] and part['all_cleared'] and part['correct_absence'] - and part['normal_exit'] and not part['anomalies'] - and not part['timing_mismatches'] - for row in rows for part in (row[baseline],row['rl'])) - report[key + "_summary"] = { - "mean_virtual_s": { - baseline: statistics.mean(row[baseline]["virtual_time_s"] for row in rows), - "rl": statistics.mean(row["rl"]["virtual_time_s"] for row in rows), - }, - "mean_delta_s": statistics.mean(deltas), - "relative_delta_pct": 100.0 * statistics.mean(deltas) - / statistics.mean(row[baseline]["virtual_time_s"] for row in rows), - "mean_measures": { - baseline: statistics.mean(row[baseline]["measures"] for row in rows), - "rl": statistics.mean(row["rl"]["measures"] for row in rows), - }, - "all_cleared": cleared, - "all_valid": valid, - } - output = Path(args.output) - output.parent.mkdir(parents=True, exist_ok=True) - output.write_text(json.dumps(report, ensure_ascii=False, indent=2), - encoding="utf-8") - print(json.dumps({"event": "benchmark_done", "path": str(output), - "q3": report["q3_summary"], - "q4": report["q4_summary"]}, - ensure_ascii=False, indent=2)) - return 0 if report['q3_summary']['all_valid'] and report['q4_summary']['all_valid'] else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/evaluate_rl_revision.py b/scripts/evaluate_rl_revision.py deleted file mode 100644 index 209b151..0000000 --- a/scripts/evaluate_rl_revision.py +++ /dev/null @@ -1,125 +0,0 @@ -"""同场景配对比较手写策略、旧PPO及修订模型；真值仅在评测端使用。""" -import argparse -from dataclasses import asdict -import hashlib -import json -import math -from pathlib import Path -from statistics import mean, median -from time import perf_counter - -import torch - -from mcm26b.config import ProblemConfig -from mcm26b.evaluation import recompute_virtual_time -from mcm26b.harness import LocalSimTransport, mixed_scene, random_omni_scene, referee_spec -from mcm26b.planner import DirectionalExtension -from mcm26b.planner.bayes_loop import BayesLoop -from mcm26b.planner.rl_loop import RLBayesLoop, RLDirectionalLoop -from mcm26b.protocol.client import SimulatorClient -from referee import LocalSimulator, Source - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--split', choices=('development','validation','stress'), - default='development') - parser.add_argument('--variants', default='baseline,legacy,revised') - parser.add_argument('--problems', default='3,4') - parser.add_argument('--model-dir', type=Path, default=Path('output/rl/review-20260911')) - parser.add_argument('--label', required=True) - args = parser.parse_args() - torch.set_num_threads(1) - config = ProblemConfig() - output = args.model_dir / 'evaluation' / args.split / args.label - output.mkdir(parents=True, exist_ok=False) - variants = args.variants.split(',') - rows = [] - def save(): - aggregates = {} - for problem in (3,4): - for variant in variants: - part = [r for r in rows if r['problem']==problem and r['variant']==variant] - if part: - aggregates[f'q{problem}-{variant}'] = dict( - runs=len(part), valid=sum(r['valid'] for r in part), - mean_virtual_s=mean(r['virtual_s'] for r in part), - median_virtual_s=median(r['virtual_s'] for r in part), - max_virtual_s=max(r['virtual_s'] for r in part), - failed_clears=sum(r['failed_clears'] for r in part), - repeated_measures=sum(r['repeated_measures'] for r in part)) - result = dict(source='local_referee_only', config=asdict(config), split=args.split, - rows=rows, aggregates=aggregates) - (output/'results.json').write_text( - json.dumps(result,ensure_ascii=False,indent=2),encoding='utf-8') - return aggregates - for problem in map(int, args.problems.split(',')): - scenes = [] - if args.split == 'stress': - for count in (10,16): - scenes.append((f'boundary-{count}', count, 99000+count, [ - Source(c+1,1800*math.cos(2*math.pi*c/count),1800*math.sin(2*math.pi*c/count), - 1000,directional=(problem==4 and c%2==0), - heading_deg=360*c/count if problem==4 and c%2==0 else 0) - for c in range(count)])) - else: - seeds = range(700,703) if args.split=='development' else range(91000,91006) - counts = (10,16) if args.split=='development' else (10,13,16) - for count in counts: - for seed in seeds: - directional = (1, count//2, count-1)[seed%3] - sources = (random_omni_scene(seed, count, config) if problem==3 else - mixed_scene(seed, count, directional, config)) - scenes.append((f'{count}-{seed}',count,seed,sources)) - for name,count,seed,sources in scenes: - scene_data = dict(sources=[asdict(s) for s in sources], error_seed=seed+2000000) - scene_path = output/f'q{problem}-{name}-scene.json' - scene_path.write_text(json.dumps(scene_data),encoding='utf-8') - for variant in variants: - model = None - cls = BayesLoop if problem==3 else DirectionalExtension - kwargs = {} - if variant != 'baseline': - cls = RLBayesLoop if problem==3 else RLDirectionalLoop - model = (Path(f'output/rl/q{problem}_ppo.zip') if variant=='legacy' - else args.model_dir/f'q{problem}_masked.zip') - kwargs = dict(model_path=model, device='cpu') - sim = LocalSimulator(sources,robot_id='RL-AUDIT',spec=referee_spec(config), - error_seed=seed+2000000) - client = SimulatorClient('RL-AUDIT',config=config,transport=LocalSimTransport(sim)) - start = perf_counter() - summary = cls(client,config,**kwargs).run() - events = client.log.action_events() - truth = {s.channel for s in sources} - successful = {e['request']['channel'] for e in events - if e['response'].get('clear_result')=='success'} - seen = set() - repeats = 0 - for e in events: - if e['path']=='/measure': - p=e['request']['position'] - tag=(e['request']['channel'],round(p['x'],6),round(p['y'],6)) - repeats += tag in seen - seen.add(tag) - mismatches = recompute_virtual_time(events)['mismatches'] - valid = (successful==truth and summary['completed'] and not summary['anomalies'] - and not set(summary['absent_channels']) & truth - and summary['exit_reason']=='user_exit' and not mismatches) - row = dict(case=name,problem=problem,variant=variant,valid=valid, - virtual_s=client.state.virtual_time_s,all_cleared=successful==truth, - measures=summary['measures'], - failed_clears=summary['clears']-len(successful), - repeated_measures=repeats,wall_s=perf_counter()-start, - model_sha256=(hashlib.sha256(model.read_bytes()).hexdigest() - if model else None)) - rows.append(row) - (output/f'q{problem}-{name}-{variant}.json').write_text(json.dumps( - dict(row=row,summary=summary,events=events,timing_mismatches=mismatches), - ensure_ascii=False),encoding='utf-8') - save() - print(json.dumps(row),flush=True) - print(json.dumps(save(),indent=2),flush=True) - - -if __name__=='__main__': - main() diff --git a/scripts/pretrain_hrl_expert.py b/scripts/pretrain_hrl_expert.py deleted file mode 100644 index f784984..0000000 --- a/scripts/pretrain_hrl_expert.py +++ /dev/null @@ -1,24 +0,0 @@ -"""高层专家轨迹预训练入口（P2 待实现）。 - -按 docs/hrl-sol-execution.md，P2 才运行 `hrl_heuristic` 收集 -(episode_id, state, candidates, mask, action)，再按 episode 划分训练/验证集做 BC。 -本文件在 P0+P1 阶段只提供命令行占位，避免把未实现的 BC 说成已交付。 -""" -from __future__ import annotations - -import argparse - - -def main(argv=None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--problem", type=int, choices=(3, 4), default=3) - parser.add_argument("--episodes", type=int, default=3000) - parser.add_argument("--output", default="output/hrl/bc_high") - parser.parse_args(argv) - raise NotImplementedError( - "P2 尚未实现：专家轨迹收集/BC 需要先通过 P1 smoke；" - "当前请使用 scripts/benchmark_hrl.py --variants hrl_heuristic 做启发式骨架验证。") - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/pretrain_rl_expert.py b/scripts/pretrain_rl_expert.py deleted file mode 100644 index 7a3507c..0000000 --- a/scripts/pretrain_rl_expert.py +++ /dev/null @@ -1,255 +0,0 @@ -"""用自建模拟器的既有专家逻辑做行为克隆，初始化 PPO 策略网络。 - -思路：自建 referee 的观测模型与真值逻辑既快又可复现。先在它上面用 -手写专家（Q3: worst-posterior 选点器；Q4: 遮挡梯子选点器）生成 -(观测, 专家动作) 数据集，训练一个与 PPO policy_net 同构的 MLP 分类器， -再把权重拷入 PPO 模型作为初始化；随后 scripts/train_rl.py --init 继续 -PPO 微调。这样 RL 从强先验出发，而不是从随机策略开始。 - -产物： - output/rl/bc/q3_expert.npz output/rl/bc/q3_bc.pt output/rl/q3_ppo_bc_init.zip - output/rl/bc/q4_expert.npz output/rl/bc/q4_bc.pt output/rl/q4_ppo_bc_init.zip -""" -from __future__ import annotations - -import argparse -import json -import math -import time -from pathlib import Path - -import numpy as np -import torch -from stable_baselines3 import PPO - -from mcm26b.config import ProblemConfig -from mcm26b.planner import safe_disk_radius, select_measurement_point -from mcm26b.rl._constants import NUM_ACTIONS, OBS_DIM -from mcm26b.rl.env import LocalizationEnv -from mcm26b.rl.features import candidate_actions - -HIDDEN = (128, 128) - - -def expert_action(env: LocalizationEnv) -> int: - """当前状态的专家动作编号（与训练环境候选同源，就近映射）。""" - config = env.config - channel = env._channel - region = env._belief.candidate_set(channel) - position = env._client.state.position - blocked = env._blocked - candidates = candidate_actions(region, position, config, blocked) - - if env.problem == 3: - expert_point = select_measurement_point( - region, position, config).position - else: - center, radius = region.mec() - safe = safe_disk_radius(region, config) - if not blocked: - expert_point = select_measurement_point( - region, position, config).position - elif len(blocked) == 1 and safe >= 0: - first = blocked[0] - dx, dy = center[0] - first[0], center[1] - first[1] - norm = math.hypot(dx, dy) - if norm <= 1e-9: - expert_point = (center[0] + safe, center[1]) - else: - scale = min(1.0, safe / norm) - expert_point = (center[0] + dx * scale, - center[1] + dy * scale) - else: - first = blocked[0] - base = math.atan2(first[1] - center[1], first[0] - center[0]) - angle = base + math.radians(45.0 * (len(blocked) - 1)) - expert_point = (center[0] + radius * math.cos(angle), - center[1] + radius * math.sin(angle)) - return min(range(len(candidates)), - key=lambda k: math.dist(candidates[k], expert_point)) - - -class BCMlp(torch.nn.Module): - def __init__(self): - super().__init__() - self.net = torch.nn.Sequential( - torch.nn.Linear(OBS_DIM, HIDDEN[0]), torch.nn.Tanh(), - torch.nn.Linear(HIDDEN[0], HIDDEN[1]), torch.nn.Tanh(), - torch.nn.Linear(HIDDEN[1], NUM_ACTIONS)) - - def forward(self, x): - return self.net(x) - - -def collect(problem: int, episodes: int, seed: int, config: ProblemConfig): - env = LocalizationEnv( - config, problem=problem, - directional_probability=1.0 if problem == 4 else 0.0, - max_episode_steps=16) - observations, actions, groups = [], [], [] - for episode in range(episodes): - obs, _info = env.reset(seed=seed + episode) - for _step in range(env.max_episode_steps + 2): - action = expert_action(env) - observations.append(obs) - actions.append(action) - groups.append(episode) - obs, _reward, terminated, truncated, _info = env.step(action) - if terminated or truncated: - break - return np.asarray(observations, dtype=np.float32), \ - np.asarray(actions, dtype=np.int64), np.asarray(groups, dtype=np.int64) - - -def train_bc(x, y, *, epochs: int, batch_size: int, lr: float, - device: str, groups, valid_frac: float = 0.1): - model = BCMlp().to(device) - optimizer = torch.optim.AdamW(model.parameters(), lr=lr) - episodes = np.unique(groups) - if len(episodes) < 2: - raise ValueError('行为克隆至少需要两个独立episode才能划分验证集') - valid_groups = np.random.default_rng(0).permutation(episodes) - valid_groups = valid_groups[:max(1, int(len(episodes)*valid_frac))] - valid = torch.as_tensor(np.isin(groups, valid_groups), device=device) - x_t, y_t = torch.as_tensor(x, device=device), torch.as_tensor(y, device=device) - x_v, y_v = x_t[valid], y_t[valid] - x_tr, y_tr = x_t[~valid], y_t[~valid] - loss_fn = torch.nn.CrossEntropyLoss() - for epoch in range(epochs): - order = torch.randperm(len(x_tr), device=device) - total = correct = 0.0 - for start in range(0, len(order), batch_size): - idx = order[start:start + batch_size] - logits = model(x_tr[idx]) - loss = loss_fn(logits, y_tr[idx]) - optimizer.zero_grad() - loss.backward() - optimizer.step() - total += float(loss.item()) * len(idx) - correct += int((logits.argmax(1) == y_tr[idx]).sum().item()) - train_acc = correct / len(x_tr) - with torch.no_grad(): - valid_acc = float((model(x_v).argmax(1) == y_v).float().mean().item()) - if epoch == 0 or (epoch + 1) % 5 == 0: - print(json.dumps({"event": "bc_epoch", "epoch": epoch + 1, - "loss": round(total / len(x_tr), 4), - "train_acc": round(train_acc, 4), - "valid_acc": round(valid_acc, 4)}, - ensure_ascii=False)) - return model, float(valid_acc) - - -def copy_into_ppo(problem: int, model: BCMlp, device: str) -> Path | None: - """把 BC 网络权重按 SB3 的 policy_net/action_net 拆分拷入 PPO。""" - # 当前奖励/特征已修订，重新初始化critic，不能沿用旧任务的价值预测。 - env = LocalizationEnv(problem=problem) - ppo = PPO('MlpPolicy', env, policy_kwargs={'net_arch': list(HIDDEN)}, - n_steps=512, batch_size=128, gamma=1.0, device=device) - state = {key.removeprefix("net."): value - for key, value in model.state_dict().items()} - # SB3 MlpPolicy：policy_net 只含前两个 Linear（Tanh 激活）， - # 输出层在 action_net。BC 的 Sequential 键为 0/2/4。 - policy_state = {key: value for key, value in state.items() - if key in ("0.weight", "0.bias", "2.weight", "2.bias")} - action_state = { - "weight": state["4.weight"], "bias": state["4.bias"]} - ppo.policy.mlp_extractor.policy_net.load_state_dict(policy_state) - ppo.policy.action_net.load_state_dict(action_state) - # 数值检查整个前向映射，而不只是线性层形状。 - probe = torch.randn((8, OBS_DIM), device=device) - with torch.no_grad(): - copied = ppo.policy.action_net(ppo.policy.mlp_extractor.policy_net(probe)) - if not torch.allclose(model(probe), copied, atol=1e-6): - raise RuntimeError('行为克隆与PPO激活函数/前向映射不一致') - init_path = Path(f"output/rl/q{problem}_ppo_bc_init.zip") - ppo.save(str(init_path)) - env.close() - init_path.with_suffix('.json').write_text(json.dumps( - dict(algorithm='ppo', feature_version=2, problem=problem)), encoding='utf-8') - return init_path - - -def main(argv=None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--problem", type=int, choices=(3, 4), required=True) - parser.add_argument("--episodes", type=int, default=3000) - parser.add_argument("--epochs", type=int, default=20) - parser.add_argument("--batch-size", type=int, default=256) - parser.add_argument("--lr", type=float, default=3e-4) - parser.add_argument("--seed", type=int, default=20260911) - parser.add_argument("--device", default="auto") - parser.add_argument("--copy-only", action="store_true", - help="跳过采集/训练，只把已有 q_bc.pt 拷入 PPO 初始化") - args = parser.parse_args(argv) - - if args.device == "auto": - device = "cuda" if torch.cuda.is_available() else "cpu" - else: - device = args.device - print(json.dumps({"event": "bc_start", "problem": args.problem, - "episodes": args.episodes, "device": device, - "cuda": torch.cuda.is_available()}, - ensure_ascii=False)) - - if args.copy_only: - bc_dir = Path("output/rl/bc") - checkpoint = torch.load(bc_dir / f"q{args.problem}_bc.pt", - map_location="cpu") - if checkpoint.get('activation') != 'Tanh' or checkpoint.get('feature_version') != 2: - parser.error('旧ReLU/旧特征BC检查点不能直接拷贝，请重新采集并训练') - model = BCMlp() - model.load_state_dict(checkpoint["state_dict"]) - init_path = copy_into_ppo(args.problem, model, device) - dataset = np.load(bc_dir / f"q{args.problem}_expert.npz", - allow_pickle=False) - report = { - "problem": args.problem, - "samples": int(len(dataset["obs"])), - "valid_accuracy": checkpoint.get("valid_acc"), - "dataset": str(bc_dir / f"q{args.problem}_expert.npz"), - "bc_model": str(bc_dir / f"q{args.problem}_bc.pt"), - "ppo_init": str(init_path) if init_path is not None else None, - } - (bc_dir / f"q{args.problem}_bc.json").write_text( - json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") - print(json.dumps({"event": "bc_copy", "problem": args.problem, - "ppo_init": str(init_path), **report}, - ensure_ascii=False, indent=2)) - return 0 if init_path is not None else 2 - - config = ProblemConfig() - started = time.monotonic() - torch.manual_seed(args.seed) - x, y, groups = collect(args.problem, args.episodes, args.seed, config) - collect_s = time.monotonic() - started - bc_dir = Path("output/rl/bc") - bc_dir.mkdir(parents=True, exist_ok=True) - np.savez_compressed(bc_dir / f"q{args.problem}_expert.npz", obs=x, action=y, episode=groups) - - model, valid_acc = train_bc(x, y, epochs=args.epochs, - batch_size=args.batch_size, lr=args.lr, - device=device, groups=groups) - torch.save({"state_dict": model.state_dict(), "obs_dim": OBS_DIM, - "actions": NUM_ACTIONS, "valid_acc": valid_acc, - "activation": "Tanh", "feature_version": 2}, - bc_dir / f"q{args.problem}_bc.pt") - - # 拷入现有 PPO 的 policy_net/action_net 作为初始化（critic 保持原权重）。 - init_path = copy_into_ppo(args.problem, model, device) - report = { - "problem": args.problem, "episodes": args.episodes, - "samples": int(len(x)), "valid_accuracy": valid_acc, - "collect_seconds": round(collect_s, 1), - "train_seconds": round(time.monotonic() - started - collect_s, 1), - "dataset": str(bc_dir / f"q{args.problem}_expert.npz"), - "bc_model": str(bc_dir / f"q{args.problem}_bc.pt"), - "ppo_init": str(init_path) if init_path is not None else None, - } - (bc_dir / f"q{args.problem}_bc.json").write_text( - json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") - print(json.dumps(report, ensure_ascii=False, indent=2)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/save_official_rl_summary.py b/scripts/save_official_rl_summary.py deleted file mode 100644 index 8443a40..0000000 --- a/scripts/save_official_rl_summary.py +++ /dev/null @@ -1,36 +0,0 @@ -"""从 Tee 输出的 CLI 文本中提取末尾 JSON，落盘为 summary.json。 - -官方演练目录由 run 命令产生 cli-output.txt（前面可能有 CUDA 警告）， -本脚本只提取最后一个完整 JSON 对象，并核对 adjacent 的官方 result.json。 -""" -import argparse -import json -from pathlib import Path - - -def extract(text: str) -> dict: - start = text.rfind("\n{") - if start < 0: - raise ValueError("未找到 JSON 对象") - return json.loads(text[start + 1:]) - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("folder", type=Path) - args = parser.parse_args() - folder = args.folder - summary = extract((folder / "cli-output.txt").read_text(encoding="utf-8")) - (folder / "summary.json").write_text( - json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - print(json.dumps({ - "folder": str(folder), "completed": summary["completed"], - "virtual_time_s": summary["virtual_time_s"], - "cleared_channels": summary["cleared_channels"], - "total_requests": summary["total_requests"], - }, ensure_ascii=False)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/summarize_rl_revision.py b/scripts/summarize_rl_revision.py deleted file mode 100644 index 60ba546..0000000 --- a/scripts/summarize_rl_revision.py +++ /dev/null @@ -1,78 +0,0 @@ -"""汇总配对置信区间、历史官方证据与BC/特征错位探针，不连接模拟器。""" -from pathlib import Path -import json -import hashlib -import numpy as np -import torch - -import verify_rl_practice -from mcm26b.rl.env import LocalizationEnv -from mcm26b.rl.features import build_observation - - -def main(): - root = Path('output/rl/review-20260911') - records = json.loads((root/'evaluation/validation/final/results.json') - .read_text(encoding='utf-8')) - rng = np.random.default_rng(20260911) - comparisons = {} - for problem in (3,4): - part = [r for r in records['rows'] if r['problem']==problem] - for label in ('baseline','legacy'): - baseline = {r['case']:r for r in part if r['variant']==label} - selected = sorted((r for r in part if r['variant']=='revised'),key=lambda r:r['case']) - a = np.array([baseline[r['case']]['virtual_s'] for r in selected]) - b = np.array([r['virtual_s'] for r in selected]) - samples = rng.integers(0,len(a),(10000,len(a))) - changes = (b[samples].mean(axis=1)/a[samples].mean(axis=1)-1)*100 - comparisons[f'q{problem}-revised-vs-{label}'] = dict( - relative_change_pct=(b.mean()/a.mean()-1)*100, - paired_bootstrap_95_pct=np.quantile(changes,[.025,.975]).tolist(), - faster_cases=int(sum(b/checkpoints/_latest.zip`` 原子覆盖写；训练开始前先写一份 - 起步基线。因此即使进程被 SIGKILL / ``taskkill /F`` 强杀（没有任何 Python - 收尾机会），也**最多只丢 ``--recovery-every`` 个环境步**，且磁盘上始终有一份 - 完整、可加载、可 ``--resume`` 的产物； -- **原子落盘**：模型 zip、旁挂 json 与 ``/resume.json`` 都是"临时文件 + - ``os.replace``"，保存途中被杀不会留下半截文件，也不会破坏上一份完整快照； -- ``--checkpoint-every`` 每 N 个环境步在 ``<模型目录>/checkpoints/`` 另留一份历史 - checkpoint（``_<步数>.zip``），只保留最近 ``--checkpoint-keep`` 个，供 - 返祖对比与多档恢复； -- ``/resume.json`` 是恢复的**权威指针**：无论上一刻是滚动恢复点、历史 - checkpoint、优雅收尾还是异常兜底，它总指向一份可续训产物，并记录增量语义 - （``timesteps`` 是追加步数）与生效过的 PPO 超参； -- ``--control-file`` 是训练前端的**跨平台**控制通道：前端原子写 JSON - （``paused`` / ``stop``），本脚本每个 step 轮询。暂停时训练在 step 边界挂起， - 停止时先保存模型与元数据再退出； -- 信号（SIGTERM/SIGINT，Windows 上另有 SIGBREAK）仍可用：直接 kill 本脚本或 - 前端发进程组信号时同样走优雅收尾；subproc worker 在 :func:`_make_env` 里忽略 - 这些信号，保证它们活到主进程 ``vec_env.close()`` 之后再退出； -- 任何异常都会先落一份 ``*_emergency.zip`` + ``resume.json`` 再抛出，不再出现 - "跑了很久什么都没有"。 - -为什么暂停/停止不靠信号：Windows 没有 SIGSTOP，且 ``os.kill(pid, SIGTERM)`` 会 -退化成 ``TerminateProcess`` 硬杀——收尾分支根本不会执行。控制文件两端都只是 -普通文件 IO，POSIX 与 Windows 行为一致（见 :mod:`mcm26b.process_control`）。 - -P1 smoke 示例： - python scripts/train_hrl_high.py \ - --problem 3 --stage 1 --timesteps 2048 --seed 320001 \ - --envs 1 --vec-env dummy --device mps \ - --config-profile production --model output/hrl/p1-smoke/q3.zip \ - --eval-episodes 2 --verbose 0 -""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import math -import os -import platform -import random -import signal -import sys -import time -from pathlib import Path -from typing import Any - -from mcm26b.config import ProblemConfig -from mcm26b.hrl.curriculum import STAGES -from mcm26b.hrl.constants import (ACTION_LAYOUT_VERSION, FEATURE_VERSION, - HIGH_OBS_DIM, N_CANDIDATES, REWARD_VERSION) -from mcm26b.process_control import ControlState, read_control - - -def _stage_from_index(index: int): - if not 1 <= int(index) <= len(STAGES): - raise ValueError(f"stage 必须在 1..{len(STAGES)}") - return STAGES[int(index) - 1] - - -def _profile_fidelity(profile: str) -> str: - if profile == "production": - return "production" - if profile == "coarse-pretrain": - return "fast" - raise ValueError(f"未知 config-profile: {profile}") - - -def _reward_from_profile(profile: str): - from mcm26b.hrl.hierarchical_env import HRLReward - - if profile == "time": - return HRLReward.time_only() - if profile == "prompt": - return HRLReward() - raise ValueError(f"未知 reward-profile: {profile}") - - -def _stop_signals() -> list[int]: - """本平台可用于"请求收尾"的信号集合。 - - Windows 上没有 SIGTERM 的软语义（``os.kill`` 直接 TerminateProcess），但 - ``SIGBREAK`` 可以由 ``CTRL_BREAK_EVENT`` 触发并进入 Python 处理器；POSIX 上 - SIGTERM/SIGINT 均可捕获。控制文件才是主通道，这里只是让"直接 kill 脚本" - 也尽量走优雅收尾。 - """ - signals = [signal.SIGTERM, signal.SIGINT] - sigbreak = getattr(signal, "SIGBREAK", None) - if sigbreak is not None: - signals.append(sigbreak) - return signals - - -def _ignore_stop_signals_in_worker() -> None: - """subproc worker 忽略收尾信号，保证主进程能优雅落盘。 - - 无论信号来自进程组 SIGTERM 还是 Windows ``CTRL_BREAK_EVENT``，若 worker - 同时退出，主进程会在保存模型之前就拿到断管/EOF 错误，整局训练白跑。worker - 活在管道后面，主进程收尾时通过 ``vec_env.close()`` 的 ``close`` 指令正常 - 结束它们。 - """ - try: - import multiprocessing - if multiprocessing.current_process().name == "MainProcess": - return - for signum in _stop_signals(): - signal.signal(signum, signal.SIG_IGN) - except (ImportError, ValueError, OSError): # 非主线程/平台不支持：不影响训练 - return - - -class _ControlPoll: - """控制文件轮询状态机（纯标准库，可脱离 SB3 单测）。 - - ``poll()`` 每 ``poll_every`` 次调用才真正读盘，避免每个 env step 都做一次 - 文件 IO；``wait_while_paused()`` 阻塞到暂停解除或收到 stop，并累计暂停秒数 - （``--max-wall-s`` 要把这段扣掉，否则"挂起一小时回来就判超时"）。 - """ - - def __init__(self, path: Path, *, poll_every: int = 32, - poll_s: float = 0.2): - self.path = Path(path) - self.poll_every = max(1, int(poll_every)) - self.poll_s = max(0.02, float(poll_s)) - self.paused_total_s = 0.0 - self._calls = 0 - self._state = ControlState() - - def poll(self) -> ControlState: - """按抽样间隔返回最近一次控制状态。""" - self._calls += 1 - if self._calls % self.poll_every: - return self._state - self._state = read_control(self.path) - return self._state - - def wait_while_paused(self, on_pause, on_resume) -> bool: - """阻塞到继续或停止；返回 True 表示期间收到了 stop。 - - ``on_pause`` / ``on_resume`` 用来向前端回显事件（训练脚本的 JSON 行）。 - """ - on_pause() - started = time.monotonic() - stopped = False - while True: - time.sleep(self.poll_s) - self._state = read_control(self.path) - if self._state.stop: - stopped = True - break - if not self._state.paused: - break - self.paused_total_s += time.monotonic() - started - on_resume() - return stopped - - -def _make_env(problem: int, stage_index: int, - monitor_path: str | None, profile: str, - reward_profile: str): - """顶层工厂：供 DummyVecEnv / SubprocVecEnv 调用（必须是可 pickle 的）。""" - from stable_baselines3.common.monitor import Monitor - - from mcm26b.hrl.hierarchical_env import HierarchicalEnv - - _ignore_stop_signals_in_worker() - stage = _stage_from_index(stage_index) - env = HierarchicalEnv( - problem=problem, - stage=stage, - fidelity=_profile_fidelity(profile), - max_episode_steps=300, - real_budget_s=1800.0, - reward=_reward_from_profile(reward_profile), - ) - return Monitor(env, filename=monitor_path) - - -def _build_vec_env(args, monitor_paths: list[str]): - from stable_baselines3.common.vec_env import (DummyVecEnv, SubprocVecEnv) - - from functools import partial - - factories = [ - partial(_make_env, args.problem, args.stage, - monitor_paths[rank], args.config_profile, args.reward_profile) - for rank in range(args.envs) - ] - if args.vec_env == "subproc": - # 每个 worker 独立 Monitor 文件；spawn 入口在 __main__ 保护内。 - return SubprocVecEnv(factories, start_method="spawn") - return DummyVecEnv(factories) - - -def _hash_file(path: Path) -> str: - return hashlib.sha256(path.read_bytes()).hexdigest() - - -def _policy_hyperparams(model) -> dict[str, Any]: - """记录**实际生效**的 PPO 超参。 - - ``--resume``/``--init`` 时 ``MaskablePPO.load`` 会一并恢复 checkpoint 内的 - n_steps/batch_size/n_epochs 等，命令行同名参数被忽略；把它们写进元数据， - 事后才能核对一局到底用什么配置训的。 - """ - - def _scalar(value): - # SB3 2.9 的 learning_rate/clip_range 可能是 Schedule 对象 - current = getattr(value, "current_value", None) - if callable(current): - try: - value = current() - except Exception: - return str(value) - try: - return float(value) - except (TypeError, ValueError): - return str(value) - - return { - "n_steps": int(getattr(model, "n_steps", 0) or 0), - "batch_size": int(getattr(model, "batch_size", 0) or 0), - "n_epochs": int(getattr(model, "n_epochs", 0) or 0), - "gamma": _scalar(getattr(model, "gamma", 0.0)), - "gae_lambda": _scalar(getattr(model, "gae_lambda", 0.0)), - "learning_rate": _scalar(getattr(model, "learning_rate", 0.0)), - "clip_range": _scalar(getattr(model, "clip_range", 0.0)), - "ent_coef": _scalar(getattr(model, "ent_coef", 0.0)), - "vf_coef": _scalar(getattr(model, "vf_coef", 0.0)), - "max_grad_norm": _scalar(getattr(model, "max_grad_norm", 0.0)), - } - - -def _progress_payload(model, start: int) -> dict[str, Any]: - """组装 ``--progress-json`` 的单行负载。 - - 纯函数（只读 model 的 logger / ep_info_buffer），便于脱离训练循环测试； - NaN / ±inf 一律折算成 None，保证 ``json.dumps(allow_nan=False)`` 不会炸。 - """ - values = getattr(model.logger, "name_to_value", {}) or {} - - def _num(key, default=None): - value = values.get(key) - if value is None: - return default - try: - value = float(value) - except (TypeError, ValueError): - return default - if not math.isfinite(value): - return default - return value - - ep_rewards: list[float] = [] - ep_lengths: list[float] = [] - for item in getattr(model, "ep_info_buffer", []) or []: - try: - ep_rewards.append(float(item.get("r"))) - ep_lengths.append(float(item.get("l"))) - except (TypeError, ValueError, AttributeError): - continue - return { - "event": "progress", - "timesteps": int(model.num_timesteps), - "start_timesteps": int(start), - "n_updates": int(getattr(model, "_n_updates", 0) or 0), - "ep_rew_mean": (sum(ep_rewards) / len(ep_rewards) - if ep_rewards else None), - "ep_len_mean": (sum(ep_lengths) / len(ep_lengths) - if ep_lengths else None), - "value_loss": _num("train/value_loss"), - "policy_gradient_loss": _num("train/policy_gradient_loss"), - "approx_kl": _num("train/approx_kl"), - "entropy_loss": _num("train/entropy_loss"), - "explained_variance": _num("train/explained_variance"), - "fps": _num("time/fps"), - } - - -def _model_metadata(*, args, wall_s: float, n_envs: int, - actual_timesteps: int) -> dict[str, Any]: - import numpy - import stable_baselines3 - import sb3_contrib - import torch - - stage = _stage_from_index(args.stage) - return { - "level": "high", - "problem": int(args.problem), - "stage": int(args.stage), - "stage_name": stage.name, - "algorithm": "maskable_ppo", - "feature_version": FEATURE_VERSION, - "action_layout_version": ACTION_LAYOUT_VERSION, - "reward_version": REWARD_VERSION, - "obs_dim": HIGH_OBS_DIM, - "act_dim": N_CANDIDATES, - "seed": int(args.seed), - "requested_timesteps": int(args.timesteps), - "timesteps_semantics": ("additional" if (args.resume or args.init) - else "total"), - "actual_timesteps": int(actual_timesteps), - "n_envs": int(n_envs), - "vec_env": args.vec_env, - "config_profile": args.config_profile, - "fidelity": _profile_fidelity(args.config_profile), - "reward_profile": args.reward_profile, - "local_selector": args.local_selector, - "local_model": args.local_model, - "checkpoint_every": int(getattr(args, "checkpoint_every", 0) or 0), - "checkpoint_keep": int(getattr(args, "checkpoint_keep", 0) or 0), - "checkpoint_dir": getattr(args, "checkpoint_dir", None), - "recovery_every": int(getattr(args, "recovery_every", 0) or 0), - "device": args.device, - "wall_seconds": round(wall_s, 3), - "max_wall_s": float(args.max_wall_s), - "created_unix_s": time.time(), - "python": sys.version.split()[0], - "platform": platform.platform(), - "torch": torch.__version__, - "numpy": numpy.__version__, - "stable_baselines3": stable_baselines3.__version__, - "sb3_contrib": sb3_contrib.__version__, - "mps_available": bool(getattr(getattr(torch.backends, "mps", None), - "is_available", lambda: False)()), - } - - -def _write_json_atomic(path: Path, payload: dict[str, Any]) -> None: - """原子写 JSON：同目录 ``.tmp-`` 前缀临时文件 + ``os.replace``。 - - 急停（SIGKILL / ``taskkill /F``）会绕过所有 Python 清理逻辑，保存途中被杀 - 是完全可能的。原子替换保证**真实文件名下要么是旧的完整文件、要么是新的 - 完整文件**，不会出现半截 JSON 让续训读到一半就崩。 - """ - path = Path(path) - path.parent.mkdir(parents=True, exist_ok=True) - tmp = path.with_name(f".tmp-{path.name}") - tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2), - encoding="utf-8") - os.replace(tmp, path) - - -def _save_model_atomic(model, target: Path) -> Path: - """把 SB3 模型原子地写到 ``target``，返回实际落盘路径。 - - 先写 ``.tmp-``（同样以 ``.zip`` 结尾，SB3 不会再加后缀），完整写完后 - ``os.replace`` 覆盖目标。于是： - - - 保存途中被强杀 → 目标路径仍是上一份**完整**的快照，可继续复训； - - 保存成功 → 目标路径立即变成新快照，不存在"看起来存在但打不开"的窗口。 - - 临时名带 ``.tmp-`` 前缀，因此不会被 ``*_ckpt_*.zip`` 的清理逻辑误当成正式 - checkpoint。 - """ - target = Path(target) - target.parent.mkdir(parents=True, exist_ok=True) - tmp = target.with_name(f".tmp-{target.name}") - model.save(str(tmp)) - if not tmp.exists(): # SB3 对未知后缀可能自行补 .zip - candidate = tmp.with_name(tmp.name + ".zip") - if candidate.exists(): - tmp = candidate - os.replace(tmp, target) - return target - - -def _write_resume_hint(*, run_dir: Path, model_path: Path, timesteps: int, - reason: str, model=None) -> dict[str, Any]: - """把"从哪继续"写进 ``/resume.json``。 - - 这是急停恢复的**权威指针**：无论上一刻是周期 checkpoint、滚动恢复点、 - 优雅收尾还是异常兜底，这个文件总指向一份完整、可 ``--resume`` 的产物， - 并带上续训语义（``timesteps`` 是增量步数）与生效过的 PPO 超参。 - """ - hint: dict[str, Any] = { - "event": "resume_hint", - "model": str(model_path), - "timesteps": int(timesteps), - "timesteps_semantics": "additional", - "reason": str(reason), - "created_unix_s": time.time(), - "resume_args": ["--resume", "--model", str(model_path)], - } - if model is not None: - hint["ppo_hyperparams"] = _policy_hyperparams(model) - _write_json_atomic(run_dir / "resume.json", hint) - return hint - - -def _write_metadata(*, model_path: Path, args, wall_s: float, n_envs: int, - actual_timesteps: int, monitor_paths: list[str], - model=None, - extra: dict[str, Any] | None = None) -> dict[str, Any]: - """写模型旁挂元数据（含 sha256），训练前端/审计都读它。""" - metadata = _model_metadata(args=args, wall_s=wall_s, n_envs=n_envs, - actual_timesteps=actual_timesteps) - if model is not None: - metadata["ppo_hyperparams"] = _policy_hyperparams(model) - metadata["model_sha256"] = _hash_file(model_path) - metadata["monitor_files"] = list(monitor_paths) - metadata["reward_note"] = ( - "reward_profile=time 使用 HRLReward.time_only（P0.4 纯时间口径）；" - "reward_profile=prompt 使用原始 §5.2 塑形。模型元数据已固化口径。") - if extra: - metadata.update(extra) - _write_json_atomic(model_path.with_suffix(".json"), metadata) - return metadata - - -def _prune_checkpoints(directory: Path, stem: str, keep: int) -> list[str]: - """只保留最近 ``keep`` 个 checkpoint（zip+json 同删），返回被删的 zip 路径。 - - 只匹配 ``_<步数>.zip``；滚动恢复点 ``_latest.zip`` 与临时文件 - ``.tmp-*.zip`` 都不匹配，因此永远不会被历史清理误删。 - """ - files = sorted((path for path in directory.glob(f"{stem}_[0-9]*.zip") - if path.stem.rsplit("_", 1)[-1].isdigit()), - key=lambda path: path.stat().st_mtime, reverse=True) - removed: list[str] = [] - for stale in files[max(1, int(keep)):]: - for candidate in (stale, stale.with_suffix(".json")): - if candidate.exists(): - candidate.unlink() - removed.append(str(stale)) - return removed - - -def _evaluate_env(model, *, problem: int, stage_index: int, - episodes: int, seed: int, profile: str, - reward_profile: str) -> dict[str, Any]: - from mcm26b.hrl.hierarchical_env import HierarchicalEnv - - stage = _stage_from_index(stage_index) - env = HierarchicalEnv(problem=problem, stage=stage, - fidelity=_profile_fidelity(profile), - max_episode_steps=300, real_budget_s=1800.0, - reward=_reward_from_profile(reward_profile)) - wins = 0 - rows = [] - if episodes <= 0: - env.close() - return {"episodes": 0, "successes": 0, "success_rate": None, "rows": []} - for episode in range(episodes): - obs, info = env.reset(seed=seed + episode) - done = False - steps = 0 - while not done: - action, _state = model.predict( - obs, action_masks=env.action_masks(), deterministic=True) - obs, _, terminated, truncated, info = env.step(int(action)) - done = bool(terminated or truncated) - steps += 1 - # 课程阶段的小场景只需证明全部频道 accounted；完整阶段至少应清掉 - # 场景最少的题面源数。这里不读取真值，只使用环境自身统计。 - success = bool( - info.get("accounted", 0) >= 20 - and info.get("cleared", 0) >= stage.sources_min - and not info.get("anomalies") - ) - wins += int(success) - rows.append({ - "episode": episode, - "seed": seed + episode, - "steps": steps, - "cleared": info.get("cleared", 0), - "accounted": info.get("accounted", 0), - "virtual_time_s": info.get("episode_virtual_cost_s"), - "stop_reason": info.get("stop_reason"), - "anomalies": info.get("anomalies"), - "success": success, - }) - env.close() - return { - "episodes": len(rows), - "successes": wins, - "success_rate": wins / len(rows) if rows else 0.0, - "rows": rows, - } - - -def _evaluate_manifest(model_path: Path, manifest_path: str, *, - limit: int) -> dict[str, Any]: - """可选：用 benchmark_hrl 的同一套真值隔离评测在固定 manifest 上跑已训模型。""" - scripts_dir = Path(__file__).resolve().parent - if str(scripts_dir) not in sys.path: - sys.path.insert(0, str(scripts_dir)) - import benchmark_hrl as bench # scripts/benchmark_hrl.py - - manifest = json.loads(Path(manifest_path).read_text(encoding="utf-8")) - scenes = bench.scenes_from_manifest(manifest) - if limit > 0: - scenes = scenes[:limit] - rows = [] - for scene in scenes: - run_dir = Path(model_path).parent / "eval-manifest" / scene.scene_id - result = bench.run_scene_variant( - scene, "hrl", config=ProblemConfig(), - hrl_model=str(model_path), run_dir=run_dir) - rows.append(result["row"]) - aggregate = bench._aggregate(rows) - return aggregate - - -def run_training(args) -> int: - import numpy as np - import torch - from sb3_contrib import MaskablePPO - from stable_baselines3.common.callbacks import BaseCallback - from stable_baselines3.common.utils import set_random_seed - - torch.set_num_threads(1) - set_random_seed(args.seed) - random.seed(args.seed) - np.random.seed(args.seed) - if torch.cuda.is_available(): - torch.cuda.manual_seed_all(args.seed) - - model_path = Path(args.model) - if model_path.exists() and not (args.resume or args.force): - print(f"模型已存在：{model_path}；如需覆盖请加 --force，续训请加 --resume。", - file=sys.stderr) - return 2 - model_path.parent.mkdir(parents=True, exist_ok=True) - run_dir = model_path.parent - monitor_paths = [ - str(run_dir / f"high_curve_{model_path.stem}_env{rank}.monitor.csv") - for rank in range(args.envs) - ] - for path in monitor_paths: - Path(path).parent.mkdir(parents=True, exist_ok=True) - - vec_env = _build_vec_env(args, monitor_paths) - - if args.resume: - if not model_path.exists(): - print(f"--resume 指定的模型不存在：{model_path}", file=sys.stderr) - return 2 - model = MaskablePPO.load(str(model_path), env=vec_env, - device=args.device, - print_system_info=False) - elif args.init: - init_path = Path(args.init) - if not init_path.exists(): - print(f"--init 指定的模型不存在：{init_path}", file=sys.stderr) - return 2 - model = MaskablePPO.load(str(init_path), env=vec_env, - device=args.device, - print_system_info=False) - else: - model = MaskablePPO( - "MlpPolicy", - vec_env, - policy_kwargs={"net_arch": [128, 128]}, - n_steps=args.rollout_steps, - batch_size=args.batch_size, - n_epochs=args.n_epochs, - gamma=1.0, - gae_lambda=0.95, - learning_rate=3e-4, - clip_range=0.2, - ent_coef=0.005, - vf_coef=0.5, - max_grad_norm=0.5, - seed=args.seed, - device=args.device, - verbose=args.verbose, - ) - if args.init: - pass # 已在上面处理；此分支仅保持语义清晰 - - # resume/init 会恢复 checkpoint 内的 PPO 超参，命令行同名三项目前被忽略； - # 显式说出差异，避免"以为换了 n_steps 其实没换"。 - if args.resume or args.init: - effective = _policy_hyperparams(model) - requested = {"n_steps": int(args.rollout_steps), - "batch_size": int(args.batch_size), - "n_epochs": int(args.n_epochs)} - if {key: effective[key] for key in requested} != requested: - print(json.dumps({ - "event": "hyperparams_from_checkpoint", - "effective": {key: effective[key] for key in requested}, - "requested": requested, - "note": ("resume/init 沿用 checkpoint 内的 PPO 超参，" - "命令行 rollout-steps/batch-size/n-epochs 被忽略")}, - ensure_ascii=False), file=sys.stderr, flush=True) - - # ------------------------------------------------------ 优雅暂停 / 停止 - # 主通道是控制文件（跨平台）；信号是兜底：直接 kill 本进程时也走同一段收尾。 - stop_state: dict[str, Any] = {"requested": False, "signal": None} - previous_handlers: dict[int, Any] = {} - - def _request_stop(source: str) -> None: - if stop_state["requested"]: - return - stop_state["requested"] = True - stop_state["signal"] = source - print(json.dumps({"event": "stop_requested", "signal": source}), flush=True) - - def _on_stop_signal(signum, _frame): - _request_stop(signal.Signals(signum).name) - - for _signum in _stop_signals(): - try: - previous_handlers[_signum] = signal.signal(_signum, _on_stop_signal) - except (ValueError, OSError): # 非主线程/平台不支持：不阻塞训练 - pass - - class _ControlChannel(BaseCallback): - """把 :class:`_ControlPoll` 接到 SB3 的 step 回调上。 - - 为什么不用信号：Windows 没有 SIGSTOP，且 ``os.kill(SIGTERM)`` 会退化成 - ``TerminateProcess`` 硬杀，收尾分支根本不会执行。控制文件在 POSIX 与 - Windows 上都只是普通文件 IO，语义完全一致。 - - 暂停是**协作式**的：在 ``_on_step`` 内阻塞轮询，训练因此停在 step 边界， - 进程仍存活、显存仍占用，但不消耗算力；收到 stop 立刻转入收尾落盘。 - """ - - def __init__(self, path: Path, *, poll_every: int = 32, - poll_s: float = 0.2): - super().__init__() - self.poller = _ControlPoll(path, poll_every=poll_every, - poll_s=poll_s) - - def paused_seconds(self) -> float: - return self.poller.paused_total_s - - def _event(self, name: str) -> None: - print(json.dumps({"event": name, - "timesteps": int(self.model.num_timesteps)}), - flush=True) - - def _on_step(self) -> bool: - state = self.poller.poll() - if state.stop: - _request_stop("control_file") - return False - if state.paused: - stopped = self.poller.wait_while_paused( - lambda: self._event("paused"), - lambda: self._event("resumed")) - if stopped: - _request_stop("control_file") - return False - return not stop_state["requested"] - return True - - control_path = getattr(args, "control_file", None) - control_cb = _ControlChannel(Path(control_path)) if control_path else None - - class _GracefulStop(BaseCallback): - """收到停止请求后在最近一次 step 边界结束 rollout，让模型能落盘。""" - - def _on_step(self) -> bool: - if stop_state["requested"]: - print(json.dumps({"event": "graceful_stop", - "signal": stop_state["signal"], - "timesteps": int(self.model.num_timesteps)}), - flush=True) - return False - return True - - class _WallClockStop(BaseCallback): - """墙钟硬上限；暂停时间不计入，避免"挂起一小时回来就判超时"。""" - - def __init__(self, max_wall_s: float, paused_clock=None): - super().__init__() - self.max_wall_s = max_wall_s - self.start = time.monotonic() - self.paused_clock = paused_clock - - def paused_seconds(self) -> float: - return float(self.paused_clock()) if self.paused_clock else 0.0 - - def _on_step(self) -> bool: - elapsed = time.monotonic() - self.start - self.paused_seconds() - if elapsed >= self.max_wall_s: - print(json.dumps({"event": "wall_clock_stop", - "elapsed_s": elapsed})) - return False - return True - - class _ProgressJSON(BaseCallback): - """向训练前端输出单行 JSON 进度；不影响 SB3 自身 logger。 - - 前端首个 curve 点越早出现越好，因此开头几步按更小间隔发送（前 8 个 - 进度点逐步放大到 ``every_steps``），之后稳定在 ``every_steps``。 - 这样短程 smoke 也能看到完整曲线，长训不会刷屏。 - """ - - def __init__(self, every_steps: int): - super().__init__() - self.every_steps = max(1, int(every_steps)) - # init/resume 时计数器从上一阶段继续，前端要用增量算进度 - self.start = int(model.num_timesteps) - self.last_emit = self.start - self.emitted = 0 - - def _next_gap(self) -> int: - warmup = max(0.2, 1.0 - self.emitted / 8.0) - return max(1, min(self.every_steps, int(round(self.every_steps * warmup)))) - - def _on_step(self) -> bool: - current = int(self.model.num_timesteps) - if current - self.last_emit < self._next_gap(): - return True - self.last_emit = current - self.emitted += 1 - payload = _progress_payload(self.model, self.start) - print(json.dumps(payload, ensure_ascii=False, allow_nan=False), - flush=True) - return True - - started = time.monotonic() - checkpoints_dir = (Path(args.checkpoint_dir) if args.checkpoint_dir - else run_dir / "checkpoints") - checkpoint_stem = f"{model_path.stem}_ckpt" - - class _CheckpointJSON(BaseCallback): - """可续训落盘：滚动恢复点 + 周期 checkpoint + resume.json 指针。 - - - **滚动恢复点** ``_latest.zip``：每 ``recovery_every`` 个环境步 - 原子覆盖写一次，训练开始前也先写一份。它保证"急停"（SIGKILL / - ``taskkill /F``，没有任何 Python 清理机会）之后**最多只丢 - ``recovery_every`` 个环境步**，而且文件始终完整可加载。 - - **周期 checkpoint** ``_<步数>.zip``：每 ``every_steps`` 步留一份， - 只保留最近 ``keep`` 个，用于返祖对比与多档恢复。 - - 每次落盘都刷新 ``/resume.json``：无论怎么停，都有一份权威的 - "从哪继续"指针（含增量语义与生效过的 PPO 超参）。 - """ - - def __init__(self, every_steps: int, keep: int, recovery_every: int): - super().__init__() - self.every_steps = max(0, int(every_steps)) - self.keep = max(1, int(keep)) - self.recovery_every = max(0, int(recovery_every)) - self.last_saved = int(model.num_timesteps) - self.last_recovery = int(model.num_timesteps) - self.saved: list[str] = [] - self.latest_path = checkpoints_dir / f"{checkpoint_stem}_latest.zip" - - def write_recovery(self, reason: str) -> None: - """立刻写一份滚动恢复点；训练开始前调用即"起步基线"。 - - 用闭包里的 ``model`` 而不是 ``self.model``：``BaseCallback.model`` - 要等 ``learn()`` 时才被赋值，而起步基线恰恰在 ``learn()`` 之前写。 - """ - current = int(model.num_timesteps) - self.last_recovery = current - self._write_recovery(current, reason) - - def _on_step(self) -> bool: - current = int(model.num_timesteps) - if (self.recovery_every > 0 - and current - self.last_recovery >= self.recovery_every): - self.last_recovery = current - self._write_recovery(current, "interval") - if (self.every_steps > 0 - and current - self.last_saved >= self.every_steps): - self.last_saved = current - self._write_interval(current) - return True - - def _persist(self, target: Path, current: int, reason: str, - extra: dict[str, Any]) -> None: - """原子落盘 zip + 元数据，并刷新 resume.json 指针。""" - _save_model_atomic(model, target) - _write_metadata( - model_path=target, args=args, - wall_s=time.monotonic() - started, n_envs=args.envs, - actual_timesteps=current, monitor_paths=monitor_paths, - model=model, - extra={"checkpoint_timesteps": current, - "checkpoint_reason": reason, - "checkpoint_of": str(model_path), **extra}) - _write_resume_hint(run_dir=run_dir, model_path=target, - timesteps=current, reason=reason, - model=model) - - def _write_recovery(self, current: int, reason: str) -> None: - checkpoints_dir.mkdir(parents=True, exist_ok=True) - self._persist(self.latest_path, current, reason, - {"recovery": True}) - print(json.dumps({"event": "recovery_checkpoint", - "reason": reason, "timesteps": current, - "model": str(self.latest_path)}, - ensure_ascii=False), flush=True) - - def _write_interval(self, current: int) -> None: - checkpoints_dir.mkdir(parents=True, exist_ok=True) - target = checkpoints_dir / f"{checkpoint_stem}_{current:09d}.zip" - self._persist(target, current, "interval", {"checkpoint": True}) - removed = _prune_checkpoints(checkpoints_dir, checkpoint_stem, - self.keep) - self.saved.append(str(target)) - print(json.dumps({"event": "checkpoint", "timesteps": current, - "model": str(target), "pruned": removed}, - ensure_ascii=False), flush=True) - - progress_every = (args.progress_every if args.progress_every > 0 - else args.rollout_steps * args.envs) - checkpoint_cb = _CheckpointJSON(args.checkpoint_every, args.checkpoint_keep, - args.recovery_every) - if args.recovery_every > 0: - # 起步基线：训练还没跑就急停，也至少有一份可加载、可续训的产物 - checkpoint_cb.write_recovery("baseline") - # 顺序有语义：控制通道最先（暂停/停止要在本步立刻生效），进度次之， - # 墙钟上限要能读到控制通道累计的暂停秒数，checkpoint 放最后。 - callbacks: list[Any] = [] - if control_cb is not None: - callbacks.append(control_cb) - if args.progress_json: - callbacks.append(_ProgressJSON(progress_every)) - callbacks.append(_GracefulStop()) - callbacks.append(_WallClockStop( - args.max_wall_s, - paused_clock=(control_cb.paused_seconds if control_cb is not None - else None))) - callbacks.append(checkpoint_cb) - - actual_timesteps = int(model.num_timesteps) - try: - model.learn(total_timesteps=args.timesteps, - callback=callbacks if len(callbacks) > 1 else callbacks[0], - progress_bar=False, - reset_num_timesteps=not (args.resume or args.init)) - except BaseException as exc: # 训练失败也必须留下可续训产物，不静默 - failure = { - "event": "training_failed", - "error": f"{type(exc).__name__}: {exc}", - "model": str(model_path), - "timesteps": int(model.num_timesteps), - } - emergency = run_dir / f"{model_path.stem}_emergency.zip" - try: - _save_model_atomic(model, emergency) - _write_metadata(model_path=emergency, args=args, - wall_s=time.monotonic() - started, - n_envs=args.envs, - actual_timesteps=int(model.num_timesteps), - monitor_paths=monitor_paths, model=model, - extra={"emergency": True, - "error": failure["error"]}) - _write_resume_hint(run_dir=run_dir, model_path=emergency, - timesteps=int(model.num_timesteps), - reason="emergency", model=model) - failure["emergency_model"] = str(emergency) - except Exception as save_exc: # 落盘失败不能再掩盖原始异常 - failure["emergency_save_error"] = ( - f"{type(save_exc).__name__}: {save_exc}") - _write_json_atomic(run_dir / "training_failure.json", failure) - print(json.dumps(failure, ensure_ascii=False), file=sys.stderr, - flush=True) - raise - wall_s = time.monotonic() - started - actual_timesteps = int(model.num_timesteps) - stopped_by_signal = bool(stop_state["requested"]) - _save_model_atomic(model, model_path) - metadata = _write_metadata( - model_path=model_path, args=args, wall_s=wall_s, n_envs=args.envs, - actual_timesteps=actual_timesteps, monitor_paths=monitor_paths, - model=model, - extra={"stopped_by_signal": stopped_by_signal, - "stop_signal": stop_state["signal"], - "checkpoint_files": checkpoint_cb.saved, - "recovery_model": str(checkpoint_cb.latest_path)}) - # 最终模型也刷新一次 resume.json：优雅收尾后"从哪继续"指向最终模型 - _write_resume_hint(run_dir=run_dir, model_path=model_path, - timesteps=actual_timesteps, - reason=("stopped" if stopped_by_signal else "finished"), - model=model) - metadata_path = model_path.with_suffix(".json") - vec_env.close() - # 收尾阶段恢复默认处理器：再次 Ctrl+C / 停止可立即强杀，不再等待评估。 - for _signum, _handler in previous_handlers.items(): - try: - signal.signal(_signum, _handler) - except (ValueError, OSError): - pass - if stopped_by_signal: - print(json.dumps({"event": "saved_after_stop", - "model": str(model_path), - "timesteps": actual_timesteps}), flush=True) - - evaluation = _evaluate_env( - model, problem=args.problem, stage_index=args.stage, - episodes=args.eval_episodes, seed=args.seed + 1_000_000, - profile=args.config_profile, reward_profile=args.reward_profile) - manifest_eval = None - if args.eval_manifest: - try: - manifest_eval = _evaluate_manifest( - model_path, args.eval_manifest, limit=args.eval_manifest_limit) - except Exception as exc: - manifest_eval = {"error": f"{type(exc).__name__}: {exc}"} - metadata["evaluation"] = evaluation - metadata["eval_manifest"] = manifest_eval - metadata_path.write_text(json.dumps(metadata, ensure_ascii=False, indent=2), - encoding="utf-8") - print(json.dumps(metadata, ensure_ascii=False, indent=2)) - return 0 - - -def main(argv=None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--problem", type=int, choices=(3, 4), required=True) - parser.add_argument("--stage", type=int, choices=(1, 2, 3, 4), - default=4) - parser.add_argument("--timesteps", type=int, required=True) - parser.add_argument("--seed", type=int, default=320001) - parser.add_argument("--envs", type=int, default=1) - parser.add_argument("--vec-env", choices=("dummy", "subproc"), default="dummy") - parser.add_argument("--device", choices=("cpu", "cuda", "mps", "auto"), - default="auto") - parser.add_argument("--config-profile", - choices=("production", "coarse-pretrain"), - default="production") - parser.add_argument("--model", required=True) - parser.add_argument("--resume", action="store_true", - help="从 --model 续训（沿用 checkpoint 内的 PPO 超参）") - parser.add_argument("--init", default=None, - help="可选：Q3→Q4 或继续预训练权重路径") - parser.add_argument("--force", action="store_true") - parser.add_argument("--eval-manifest", default=None) - parser.add_argument("--eval-manifest-limit", type=int, default=0) - parser.add_argument("--eval-episodes", type=int, default=5) - parser.add_argument("--local-selector", choices=("geometry", "rl"), - default="geometry") - parser.add_argument("--local-model", default=None) - parser.add_argument("--rollout-steps", type=int, default=256) - parser.add_argument("--batch-size", type=int, default=256) - parser.add_argument("--n-epochs", type=int, default=10) - parser.add_argument("--max-wall-s", type=float, default=1200.0, - help="训练墙钟硬上限；默认 20 分钟") - parser.add_argument("--reward-profile", choices=("time", "prompt"), - default="time", - help="time=P0.4 纯时间口径；prompt=原始 §5.2 塑形对照") - parser.add_argument("--progress-json", action="store_true", - help="向训练前端输出单行 JSON 进度（UI 自动启用）") - parser.add_argument("--control-file", default=None, - help="训练前端控制文件（JSON：paused/stop）；" - "跨平台暂停与优雅停止的主通道") - parser.add_argument("--progress-every", type=int, default=0, - help="JSON 进度间隔环境步数；0=每个 rollout") - parser.add_argument("--checkpoint-every", type=int, default=20000, - help="每 N 个环境步留一个历史 checkpoint；0=关闭（默认 20000）") - parser.add_argument("--recovery-every", type=int, default=2000, - help="每 N 个环境步原子覆盖写一份滚动恢复点 " - "<模型目录>/checkpoints/_latest.zip；" - "急停后最多丢 N 步。0=关闭（默认 2000）") - parser.add_argument("--checkpoint-keep", type=int, default=3, - help="checkpoint 保留最近 K 个（默认 3）") - parser.add_argument("--checkpoint-dir", default=None, - help="checkpoint 目录；默认 <模型目录>/checkpoints") - parser.add_argument("--verbose", type=int, default=1) - args = parser.parse_args(argv) - if args.timesteps <= 0 or args.envs <= 0: - parser.error("timesteps 与 envs 必须为正") - if args.envs * args.rollout_steps < 2: - parser.error("PPO rollout 至少需要 2 个样本") - if args.eval_episodes < 0: - parser.error("eval-episodes 不能为负") - if args.checkpoint_every < 0: - parser.error("checkpoint-every 不能为负") - if args.recovery_every < 0: - parser.error("recovery-every 不能为负") - if args.checkpoint_keep < 1: - parser.error("checkpoint-keep 至少为 1") - return run_training(args) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/train_hrl_low.py b/scripts/train_hrl_low.py deleted file mode 100644 index a1f0291..0000000 --- a/scripts/train_hrl_low.py +++ /dev/null @@ -1,266 +0,0 @@ -"""低层 PPO 训练：8 方向到达任务（BC 直线专家热启动 + PPO 微调）。 - -产物： - output/hrl/hrl_low.zip / hrl_low.json PPO 模型（部署用） - output/hrl/bc_low_expert.npz / bc_low.pt 行为克隆数据与网络 - output/hrl/hrl_low_curve_env*.monitor.csv 每个环境独立学习曲线 - -评估口径：到达成功率 + 路径效率（直线距离/实际路径长，1 为完美直线）。 -这是独立导航实验：目前低层 PPO 仅供轨迹诊断，不影响正式 HTTP 任务。 - -示例： - uv run python scripts/train_hrl_low.py --timesteps 60000 - uv run python scripts/train_hrl_low.py --skip-bc # 纯 PPO 对照 -""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import math -import time -from pathlib import Path - -from mcm26b.config import ProblemConfig -from mcm26b.hrl.constants import (FEATURE_VERSION, LOW_MAX_STEPS, - LOW_NUM_ACTIONS, LOW_OBS_DIM) -from mcm26b.hrl.low_level_env import LowLevelEnv - -HIDDEN = (64, 64) - - -def _bc_module(obs_dim: int, n_actions: int, hidden: tuple[int, ...]): - import torch - - class BCMlp(torch.nn.Module): - def __init__(self): - super().__init__() - layers = [] - prev = obs_dim - for width in hidden: - layers += [torch.nn.Linear(prev, width), torch.nn.Tanh()] - prev = width - layers.append(torch.nn.Linear(prev, n_actions)) - self.net = torch.nn.Sequential(*layers) - - def forward(self, x): - return self.net(x) - - return BCMlp - - -def collect(episodes: int, seed: int, config: ProblemConfig): - import numpy as np - env = LowLevelEnv(config=config, seed=seed) - observations, actions, episode_ids = [], [], [] - for episode in range(episodes): - obs, _info = env.reset(seed=seed + episode) - for _step in range(LOW_MAX_STEPS + 2): - observations.append(obs) - actions.append(env.expert_action()) - episode_ids.append(episode) - obs, _r, terminated, truncated, _info = env.step(env.expert_action()) - if terminated or truncated: - break - env.close() - return (np.asarray(observations, dtype=np.float32), - np.asarray(actions, dtype=np.int64), - np.asarray(episode_ids, dtype=np.int64)) - - -def bc_validation_mask(episode_ids, *, seed: int): - """按整局留出，避免同一轨迹相邻状态跨训练/验证集。""" - import numpy as np - unique = np.unique(episode_ids) - if len(unique) < 2: - raise ValueError("BC 至少需要 2 局才能按整局划分训练/验证集") - rng = np.random.default_rng(seed) - validation_ids = rng.permutation(unique)[:max(1, len(unique) // 10)] - return np.isin(episode_ids, validation_ids) - - -def train_bc(x, y, episode_ids, *, epochs: int, batch_size: int, lr: float, - device: str, seed: int): - import torch - BCMlp = _bc_module(LOW_OBS_DIM, LOW_NUM_ACTIONS, HIDDEN) - model = BCMlp().to(device) - optimizer = torch.optim.AdamW(model.parameters(), lr=lr) - valid = torch.as_tensor( - bc_validation_mask(episode_ids, seed=seed), device=device) - x_t = torch.as_tensor(x, device=device) - y_t = torch.as_tensor(y, device=device) - x_v, y_v, x_tr, y_tr = x_t[valid], y_t[valid], x_t[~valid], y_t[~valid] - loss_fn = torch.nn.CrossEntropyLoss() - for epoch in range(epochs): - order = torch.randperm(len(x_tr), device=device) - total = correct = 0.0 - for start in range(0, len(order), batch_size): - idx = order[start:start + batch_size] - logits = model(x_tr[idx]) - loss = loss_fn(logits, y_tr[idx]) - optimizer.zero_grad() - loss.backward() - optimizer.step() - total += float(loss.item()) * len(idx) - correct += int((logits.argmax(1) == y_tr[idx]).sum().item()) - if epoch == 0 or (epoch + 1) % 5 == 0: - with torch.no_grad(): - valid_acc = float((model(x_v).argmax(1) == y_v).float().mean().item()) - print(json.dumps({"event": "bc_epoch", "epoch": epoch + 1, - "loss": round(total / len(x_tr), 4), - "train_acc": round(correct / len(x_tr), 4), - "valid_acc": round(valid_acc, 4)}, - ensure_ascii=False)) - with torch.no_grad(): - valid_acc = float((model(x_v).argmax(1) == y_v).float().mean().item()) - return model, valid_acc - - -def copy_into_ppo(bc_model, device: str, env, *, seed: int, - n_steps: int = 512) -> object: - """把 BC 权重拷入 MaskablePPO（policy_net 两层 + action_net），前向校验。""" - import torch - from sb3_contrib import MaskablePPO - ppo = MaskablePPO("MlpPolicy", env, policy_kwargs={"net_arch": list(HIDDEN)}, - n_steps=n_steps, batch_size=128, n_epochs=6, - gamma=1.0, gae_lambda=0.95, learning_rate=3e-4, - clip_range=0.2, ent_coef=0.005, vf_coef=0.5, - max_grad_norm=0.5, seed=seed, device=device) - state = {key.removeprefix("net."): value - for key, value in bc_model.state_dict().items()} - policy_state = {key: value for key, value in state.items() - if key in ("0.weight", "0.bias", "2.weight", "2.bias")} - ppo.policy.mlp_extractor.policy_net.load_state_dict(policy_state) - ppo.policy.action_net.load_state_dict( - {"weight": state["4.weight"], "bias": state["4.bias"]}) - probe = torch.randn((8, LOW_OBS_DIM), device=device) - with torch.no_grad(): - copied = ppo.policy.action_net(ppo.policy.mlp_extractor.policy_net(probe)) - if not torch.allclose(bc_model(probe), copied, atol=1e-6): - raise RuntimeError("行为克隆与 PPO 前向映射不一致") - return ppo - - -def evaluate(model, *, episodes: int, seed: int, - config: ProblemConfig) -> dict: - """确定性策略评估：成功率与路径效率（对照直线）。""" - env = LowLevelEnv(config=config, seed=seed) - successes = 0 - efficiencies = [] - for episode in range(episodes): - obs, info = env.reset(seed=seed + episode) - start, goal = tuple(info["position"]), tuple(info["goal"]) - straight = math.dist(start, goal) - steps = 0 - for _ in range(LOW_MAX_STEPS + 2): - action, _state = model.predict(obs, deterministic=True) - obs, _r, terminated, truncated, info = env.step(int(action)) - steps += 1 - if terminated or truncated: - break - successes += int(terminated) - efficiencies.append(min(1.0, straight / - max(info["path_length_m"], 1e-9))) - env.close() - return {"episodes": episodes, - "success_rate": successes / episodes, - "mean_path_efficiency": sum(efficiencies) / len(efficiencies)} - - -def main(argv=None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--timesteps", type=int, default=60_000) - parser.add_argument("--seed", type=int, default=20260911) - parser.add_argument("--bc-episodes", type=int, default=600) - parser.add_argument("--bc-epochs", type=int, default=12) - parser.add_argument("--envs", type=int, default=4) - parser.add_argument("--rollout-steps", type=int, default=512, - help="每环境每次 PPO rollout 长度；短 smoke 可设 8") - parser.add_argument("--device", default="cpu") - parser.add_argument("--model", default="output/hrl/hrl_low.zip") - parser.add_argument("--eval-episodes", type=int, default=40) - parser.add_argument("--skip-bc", action="store_true", - help="跳过行为克隆（纯 PPO 对照实验用）") - parser.add_argument("--verbose", type=int, default=1) - args = parser.parse_args(argv) - if min(args.envs, args.timesteps, args.eval_episodes, - args.rollout_steps) <= 0: - parser.error("envs/timesteps/eval-episodes/rollout-steps 必须为正数") - if args.envs * args.rollout_steps < 2: - parser.error("PPO rollout 至少需要 2 个样本") - if not args.skip_bc and (args.bc_episodes < 2 or args.bc_epochs < 1): - parser.error("BC 至少需要 2 局和 1 个 epoch") - - import torch - torch.set_num_threads(1) - from sb3_contrib import MaskablePPO - from stable_baselines3.common.monitor import Monitor - from stable_baselines3.common.vec_env import DummyVecEnv - from stable_baselines3.common.utils import set_random_seed - set_random_seed(args.seed) - - model_path = Path(args.model) - model_path.parent.mkdir(parents=True, exist_ok=True) - curve_paths = [model_path.with_name( - f"{model_path.stem}_curve_env{rank}.monitor.csv") - for rank in range(args.envs)] - - def _factory(rank: int): - def _make(): - env = Monitor(LowLevelEnv(seed=args.seed + rank), - filename=str(curve_paths[rank])) - return env - return _make - - vec_env = DummyVecEnv([_factory(rank) for rank in range(args.envs)]) - started = time.monotonic() - bc_valid_acc = None - if args.skip_bc: - model = MaskablePPO("MlpPolicy", vec_env, - policy_kwargs={"net_arch": list(HIDDEN)}, - n_steps=args.rollout_steps, batch_size=128, n_epochs=6, - gamma=1.0, gae_lambda=0.95, learning_rate=3e-4, - clip_range=0.2, ent_coef=0.005, vf_coef=0.5, - max_grad_norm=0.5, seed=args.seed, - device=args.device, verbose=args.verbose) - else: - x, y, episode_ids = collect(args.bc_episodes, args.seed, ProblemConfig()) - bc_model, bc_valid_acc = train_bc( - x, y, episode_ids, epochs=args.bc_epochs, batch_size=256, - lr=3e-4, device=args.device, seed=args.seed) - model = copy_into_ppo(bc_model, args.device, vec_env, seed=args.seed, - n_steps=args.rollout_steps) - bc_dir = model_path.parent / "bc_low.pt" - torch.save(bc_model.state_dict(), str(bc_dir)) - print(json.dumps({"event": "bc_done", "samples": int(len(x))}, - ensure_ascii=False)) - model.learn(total_timesteps=args.timesteps, - progress_bar=args.verbose > 0) - elapsed_s = time.monotonic() - started - - model.save(str(model_path)) - eval_seed = args.seed + 1_000_000 - evaluation = evaluate(model, episodes=args.eval_episodes, seed=eval_seed, - config=ProblemConfig()) - report = { - "level": "low", "timesteps": args.timesteps, "seed": args.seed, - "device": args.device, "wall_seconds": round(elapsed_s, 1), - "model": str(model_path), "algorithm": "maskable_ppo", - "feature_version": FEATURE_VERSION, "bc_skipped": args.skip_bc, - "bc_valid_acc": bc_valid_acc, "evaluation_seed": eval_seed, - "monitor_files": [str(path) for path in curve_paths], - "training_role": "standalone_navigation_only", - "obs_dim": LOW_OBS_DIM, "act_dim": LOW_NUM_ACTIONS, - "model_sha256": hashlib.sha256(model_path.read_bytes()).hexdigest(), - **evaluation, - } - model_path.with_suffix(".json").write_text( - json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") - print(json.dumps(report, ensure_ascii=False, indent=2)) - vec_env.close() - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/train_hrl_ui.py b/scripts/train_hrl_ui.py deleted file mode 100644 index bc0cbb5..0000000 --- a/scripts/train_hrl_ui.py +++ /dev/null @@ -1,5 +0,0 @@ -"""启动 HRL 模型训练前端（薄入口，便于加入支撑材料命令）。""" -from mcm26b.training_ui.server import run_training_ui - -if __name__ == "__main__": - raise SystemExit(run_training_ui()) diff --git a/scripts/train_rl.py b/scripts/train_rl.py deleted file mode 100644 index 521161a..0000000 --- a/scripts/train_rl.py +++ /dev/null @@ -1,195 +0,0 @@ -"""问题3/4 强化学习选点策略训练（PPO + CUDA）。 - -训练环境 mcm26b.rl.env.LocalizationEnv 与运行闭环共享 features 观测/候选 -布局。模型输出到 output/rl/q3_ppo.zip（问题3）或 q4_ppo.zip（问题4）， -由 `mcm26b run --strategy rl` / `--q4-strategy rl` 加载。 - -示例： - uv sync --group rl --group dev - uv run python scripts/train_rl.py --problem 3 --timesteps 30000 - uv run python scripts/train_rl.py --problem 4 --timesteps 30000 - uv run mcm26b run --robot-id 2026001 --self-test --strategy rl - uv run mcm26b run --robot-id 2026001 --self-test --problem 4 --q4-strategy rl -""" - -from __future__ import annotations - -import argparse -import json -import time -from dataclasses import asdict -import hashlib -from pathlib import Path - -from stable_baselines3 import PPO -from sb3_contrib import MaskablePPO -from stable_baselines3.common.vec_env import DummyVecEnv - -from mcm26b.config import ProblemConfig -from mcm26b.rl.env import LocalizationEnv - - -def _make_env(problem: int, seed: int, config: ProblemConfig, - directional_probability: float): - def _factory(): - env = LocalizationEnv(config, problem=problem, - directional_probability=directional_probability, - real_budget_s=3600.0) - env.reset(seed=seed) - return env - return _factory - - -def evaluate(model, problem: int, *, episodes: int = 24, - directional_probability: float = 0.5, - config: ProblemConfig = ProblemConfig()) -> dict: - """独立场景评估：确定性策略的成功率、步数与平均回报。""" - env = LocalizationEnv(config, problem=problem, - directional_probability=directional_probability) - total_reward = 0.0 - successes = 0 - steps = [] - costs = [] - for episode in range(episodes): - obs, _info = env.reset(seed=1_000_000 + episode) - episode_reward = 0.0 - for step in range(env.max_episode_steps + 2): - kwargs = {'action_masks': env.action_masks()} if isinstance(model, MaskablePPO) else {} - action, _state = model.predict(obs, deterministic=True, **kwargs) - obs, reward, terminated, truncated, info = env.step(int(action)) - episode_reward += float(reward) - if terminated or truncated: - break - total_reward += episode_reward - cleared = bool(info.get("cleared")) - successes += int(cleared) - steps.append(step + 1) - costs.append(info['virtual_cost_s']) - env.close() - return { - "episodes": episodes, - "success_rate": successes / episodes, - "mean_steps": sum(steps) / episodes, - "mean_episode_reward": total_reward / episodes, - "successes": successes, - "mean_virtual_cost_s": sum(costs) / episodes, - "max_virtual_cost_s": max(costs), - } - - -def main(argv=None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--problem", type=int, choices=(3, 4), default=3, - help="训练问题：3=全向源选点，4=含定向遮挡梯子") - parser.add_argument("--timesteps", type=int, default=30000, - help="PPO 总环境步数") - parser.add_argument("--seed", type=int, default=20260911) - parser.add_argument("--envs", type=int, default=2, help="并行环境份数（CPU 侧）") - parser.add_argument("--directional-probability", type=float, default=None, - help="问题4训练场景中定向源比例（默认 1.0，先学遮挡分支）") - parser.add_argument("--device", default="auto", - help="auto=优先 CUDA；或 cuda/cpu") - parser.add_argument("--model", default=None, - help="模型输出路径（默认 output/rl/q_ppo.zip）") - parser.add_argument("--init", default=None, - help="初始模型路径（如 output/rl/q3_ppo_bc_init.zip；" - "加载后继续 PPO 微调）") - parser.add_argument("--eval-episodes", type=int, default=24) - parser.add_argument("--verbose", type=int, default=1) - parser.add_argument('--algorithm', choices=('ppo', 'maskable_ppo'), default='maskable_ppo') - parser.add_argument('--torch-threads', type=int, default=1) - args = parser.parse_args(argv) - directional_probability = (args.directional_probability - if args.directional_probability is not None - else (1.0 if args.problem == 4 else 0.0)) - if not 0.0 <= directional_probability <= 1.0: - parser.error("--directional-probability 须在 [0,1]") - - import torch - torch.set_num_threads(args.torch_threads) - if args.device == "auto": - device = "cuda" if torch.cuda.is_available() else "cpu" - else: - device = args.device - if device == "cuda" and not torch.cuda.is_available(): - parser.error("--device cuda 但当前环境检测不到 CUDA GPU") - print(json.dumps({ - "event": "train_start", "problem": args.problem, - "timesteps": args.timesteps, "device": device, - "directional_probability": directional_probability, - "torch": torch.__version__, - "gpu": torch.cuda.get_device_name(0) if device == "cuda" else None, - "cuda_available": torch.cuda.is_available(), - }, ensure_ascii=False)) - - config = ProblemConfig() - model_cls = MaskablePPO if args.algorithm == 'maskable_ppo' else PPO - vec_env = DummyVecEnv( - [_make_env(args.problem, args.seed + rank, config, - directional_probability) - for rank in range(max(1, args.envs))]) - if args.init: - metadata = json.loads(Path(args.init).with_suffix('.json').read_text(encoding='utf-8')) - if (metadata.get('algorithm', 'ppo') != args.algorithm - or metadata.get('feature_version') != 2): - parser.error('初始化模型的算法/特征版本与当前训练不符，请重训或提供v2初始化模型') - model = model_cls.load(args.init, env=vec_env, device=device) - else: - model = model_cls( - "MlpPolicy", vec_env, - policy_kwargs={"net_arch": [128, 128]}, - n_steps=512, - batch_size=128, - n_epochs=6, - gamma=1.0, - gae_lambda=0.95, - learning_rate=3e-4, - clip_range=0.2, - ent_coef=0.005, - vf_coef=0.5, - max_grad_norm=0.5, - seed=args.seed, - device=device, - verbose=args.verbose, - ) - - started = time.monotonic() - model.learn(total_timesteps=args.timesteps, - progress_bar=args.verbose > 0) - elapsed_s = time.monotonic() - started - - model_path = Path(args.model) if args.model \ - else Path("output/rl") / f"q{args.problem}_ppo.zip" - model_path.parent.mkdir(parents=True, exist_ok=True) - model.save(str(model_path)) - - evaluation = evaluate(model, args.problem, episodes=args.eval_episodes, - directional_probability=directional_probability, - config=config) - report = { - "problem": args.problem, - "timesteps": args.timesteps, - "seed": args.seed, - "device": device, - "directional_probability": directional_probability, - "wall_seconds": round(elapsed_s, 1), - "model": str(model_path), - "init": args.init, - "algorithm": args.algorithm, - "feature_version": 2, - "reward": 'negative_virtual_cost_div100_success20_failure100', - "gamma": 1.0, - "config": asdict(config), - "model_sha256": hashlib.sha256(model_path.read_bytes()).hexdigest(), - **evaluation, - } - report_path = model_path.with_suffix(".json") - report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), - encoding="utf-8") - print(json.dumps(report, ensure_ascii=False, indent=2)) - vec_env.close() - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/tune_hrl.py b/scripts/tune_hrl.py deleted file mode 100644 index b5c40eb..0000000 --- a/scripts/tune_hrl.py +++ /dev/null @@ -1,27 +0,0 @@ -"""HRL Optuna TPE 调参入口（P2 待实现）。 - -搜索空间按用户 Prompt §10，但只允许在不破坏覆盖证明/证书/预算的工程参数上搜索： -粒子数、重采样阈值、信息增益权重、移动成本权重、检测成本权重、候选点距离、 -覆盖网格间距、停止连续无信号次数、清除距离阈值。目标 = 平均定位清除时间 + -未清除惩罚，每个配置至少 5 个随机种子。 -""" -from __future__ import annotations - -import argparse - - -def main(argv=None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--problem", type=int, choices=(3, 4), default=3) - parser.add_argument("--trials", type=int, default=30) - parser.add_argument("--timesteps", type=int, default=2048) - parser.add_argument("--manifest", default="output/hrl/manifests/dev.json") - parser.add_argument("--output", default="output/hrl/p2-tuning") - parser.parse_args(argv) - raise NotImplementedError( - "P2 尚未实现：调参必须建立在 P1 smoke 与稳定基准之上，" - "且不能搜索真实误差上限或破坏覆盖证明的网格参数。") - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/verify_rl_practice.py b/scripts/verify_rl_practice.py deleted file mode 100644 index f29895b..0000000 --- a/scripts/verify_rl_practice.py +++ /dev/null @@ -1,123 +0,0 @@ -"""Offline audit of official RL practice runs (Q3/Q4); never connects to simulator. - -Checks the copied official .result.json/.jlog, our session.jsonl and summary.json: -package sha256, enter/exit, accepted timestamps inside official window, unique -request ids, clear-success channel count == official jammer_count, summary -consistency, and virtual time recomputation with integer microsecond accumulation. -""" -from __future__ import annotations - -import argparse -import hashlib -import json -import math -from datetime import datetime -from pathlib import Path - - -def save_json(path: Path, value: dict) -> None: - path.write_text(json.dumps(value, ensure_ascii=False, indent=2, - allow_nan=False) + "\n", encoding="utf-8") - - -def audit(folder: Path) -> dict: - records = [json.loads(line) for line in - (folder / "session.jsonl").read_text(encoding="utf-8").splitlines()] - events = [e for e in records if e.get("event") == "accepted"] - summary = json.loads((folder / "summary.json").read_text(encoding="utf-8")) - official_paths = list(folder.glob("*.result.json")) - if len(official_paths) != 1 or not events: - raise ValueError("Expected one official result and nonempty session evidence") - official = json.loads(official_paths[0].read_text(encoding="utf-8")) - package = official_paths[0].with_name( - official_paths[0].name.replace(".result.json", ".jlog")) - checks: dict = {} - - checks["official_problem_matches_summary"] = ( - official["problem_no"] in (3, 4) - and summary["strategy"] == ("rl" if official["problem_no"] == 3 else "rl_q4") - and 10 <= official["jammer_count"] <= 16 - and official["omnidirectional_jammer_count"] + official["directional_jammer_count"] - == official["jammer_count"]) - checks["official_package_sha256_matches"] = ( - hashlib.sha256(package.read_bytes()).hexdigest() == official["package_sha256"]) - checks["normal_enter_and_exit"] = ( - events[0]["path"] == "/enter" and events[-1]["path"] == "/exit" - and events[-1]["response"].get("exit_reason") == "user_exit") - start_ms = datetime.fromisoformat(official["window_started_at_utc"]).timestamp() * 1000 - end_ms = datetime.fromisoformat(official["ended_at_utc"]).timestamp() * 1000 - checks["responses_within_official_run_window"] = all( - start_ms <= e["response"]["real_timestamp_ms"] <= end_ms + 1 for e in events) - ids = [e["request"]["request_id"] for e in events] - checks["unique_accepted_request_ids"] = len(ids) == len(set(ids)) - clears = [e for e in events if e["path"] == "/clear"] - successes = [e for e in clears if e["response"].get("clear_result") == "success"] - cleared = sorted({e["request"]["channel"] for e in successes}) - checks["all_official_sources_cleared"] = len(cleared) == official["jammer_count"] - checks["summary_completed_and_consistent"] = ( - bool(summary["completed"]) - and sorted(summary["cleared_channels"]) == cleared - and summary["unresolved_channels"] == [] - and summary["anomalies"] == []) - - position, channel, virtual_us = (0.0, 0.0), 1, 0 - mismatches = 0 - for event in events: - request, response, path = event["request"], event["response"], event["path"] - distance, switches, action_s = 0.0, 0, 0 - if path in ("/measure", "/clear"): - target = (request["position"]["x"], request["position"]["y"]) - distance = math.dist(position, target) - position = target - if path == "/measure": - switches = int(channel != request["channel"]) - channel = request["channel"] - action_s = 5 - else: - action_s = 5 if response["clear_result"] == "success" else 3 - virtual_us += round(distance / 5 * 1_000_000) + (switches + action_s) * 1_000_000 - if round(response["virtual_time_s"] * 1_000_000) != virtual_us: - mismatches += 1 - checks["all_timestamps_match_microsecond_recomputation"] = mismatches == 0 - real_s = (events[-1]["response"]["real_timestamp_ms"] - - events[0]["response"]["real_timestamp_ms"]) / 1000 - checks["within_real_and_virtual_limits"] = ( - real_s < events[0]["response"]["remaining_real_duration_s"] - and virtual_us / 1_000_000 < events[0]["response"]["max_virtual_duration_s"]) - - report = { - "passed": all(checks.values()), - "source": "official_practice", - "strategy": summary["strategy"], - "problem": official["problem_no"], - "case_code": official["case_code"], - "official_jammer_count": official["jammer_count"], - "directional_jammer_count": official["directional_jammer_count"], - "cleared_channels": cleared, - "accepted_requests_including_enter_exit": len(events), - "measures": len([e for e in events if e["path"] == "/measure"]), - "clear_attempts": len(clears), - "clear_successes": len(successes), - "virtual_time_s": virtual_us / 1_000_000, - "real_elapsed_server_s": real_s, - "checks": checks, - "limits": "仅验证本官方演练案例；不触碰正式测试，不解读官方加密日志内容。", - } - save_json(folder / "verification.json", report) - manifest = {p.name: {"bytes": p.stat().st_size, - "sha256": hashlib.sha256(p.read_bytes()).hexdigest()} - for p in sorted(folder.iterdir()) if p.is_file() - and p.name != "manifest.json"} - save_json(folder / "manifest.json", manifest) - return report - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("folder", type=Path) - args = parser.parse_args() - result = audit(args.folder) - print(json.dumps({"passed": result["passed"], "case_code": result["case_code"], - "problem": result["problem"], "checks": result["checks"]}, - ensure_ascii=False, indent=2)) - raise SystemExit(0 if result["passed"] else 1) diff --git a/src/mcm26b/cli.py b/src/mcm26b/cli.py index fab12d3..225328e 100644 --- a/src/mcm26b/cli.py +++ b/src/mcm26b/cli.py @@ -30,7 +30,6 @@ from .protocol.log import SessionLog from .protocol.rules import EXIT_USER Q3_STRATEGIES = {"grid": OmnidirectionalLoop, "seven": SevenBatch, "bayes": BayesLoop} -Q4_STRATEGIES = {"directional": DirectionalExtension} def _load_referee(): @@ -252,44 +251,13 @@ def _run(args) -> int: log = SessionLog(path=Path(args.log)) if args.log else SessionLog() client = SimulatorClient(args.robot_id, config=config, transport=transport, log=log) if args.problem == 4: - strategy = args.q4_strategy - loop_cls = Q4_STRATEGIES.get(strategy) + strategy = "directional" + loop_cls = DirectionalExtension else: strategy = args.strategy - loop_cls = Q3_STRATEGIES.get(strategy) - high_policy = None - low_policy = None + loop_cls = Q3_STRATEGIES[strategy] try: - if strategy == "rl": - # 惰性导入：默认运行闭环不加载 numpy/torch/stable-baselines3。 - from .planner.rl_loop import RLBayesLoop, RLDirectionalLoop - loop_cls = RLBayesLoop if args.problem == 3 else RLDirectionalLoop - elif strategy == "hrl": - # P1：高层 MaskablePPO 调度。缺少模型时显式报错，不降级伪装。 - if not args.hrl_model: - raise RuntimeError( - "--strategy hrl/--q4-strategy hrl 需要 --hrl-model；" - "无模型启发式请使用 benchmark_hrl.py 的 hrl_heuristic 变体。") - from .hrl.policy import HighLevelPolicy, LowLevelPolicy - from .planner.hrl_loop import HRLLoop - high_policy = HighLevelPolicy( - args.hrl_model, device=args.hrl_device, problem=args.problem) - if args.hrl_low_model: - low_policy = LowLevelPolicy(args.hrl_low_model, device=args.hrl_device) - loop_cls = HRLLoop - if loop_cls is None: - raise RuntimeError(f"未知策略: {strategy}") - if strategy == "rl": - summary = loop_cls(client, config, model_path=args.rl_model).run() - elif strategy == "hrl": - summary = loop_cls( - client, config, problem=args.problem, high_policy=high_policy, - low_policy=low_policy, trace_executor=args.trace_low).run() - else: - summary = loop_cls(client, config).run() - except RuntimeError as exc: - print(f"策略不可用: {exc}", file=sys.stderr) - return 2 + summary = loop_cls(client, config).run() finally: if judge is not None: _settle_local_judge(judge) @@ -301,10 +269,7 @@ def _run(args) -> int: def _finalize_run(args, summary, strategy, *, truth, judge, judge_session, real_budget_s) -> int: """补齐 summary 元信息、核对真值并输出；返回进程退出码。""" - if args.problem == 4: - summary["strategy"] = "rl_q4" if strategy == "rl" else strategy - else: - summary["strategy"] = strategy + summary["strategy"] = strategy if real_budget_s is not None: summary["self_test_real_budget_s"] = real_budget_s if args.summary_steps is not None and len(summary.get("steps", [])) > args.summary_steps: @@ -349,30 +314,10 @@ def main(argv=None) -> int: run.add_argument("--log", default=None, help="会话日志 JSONL 输出路径") run.add_argument("--problem", type=int, choices=(3, 4), default=3, help="问题3=全向源，问题4=混合定向源（默认 3）") - run.add_argument("--strategy", choices=tuple(Q3_STRATEGIES) + ("rl", "hrl"), - default="bayes", + run.add_argument("--strategy", choices=tuple(Q3_STRATEGIES), default="bayes", help="问题3策略：bayes=粒子滤波+信息增益+滚动调度（默认，" "自建消融相对 seven 再降 19.5%%），seven=七点批量，" - "grid=49点网格基线，rl=PPO 强化学习选点（需先训练模型），" - "hrl=高层 MaskablePPO 分层调度（需 --hrl-model）；" - "仅 --problem 3 生效") - run.add_argument("--q4-strategy", choices=tuple(Q4_STRATEGIES) + ("rl", "hrl"), - default="directional", - help="问题4策略：directional=49点覆盖+遮挡梯子+后备清除（默认），" - "rl=PPO 强化学习选点（需先训练模型），" - "hrl=高层 MaskablePPO 分层调度（需 --hrl-model）；仅 --problem 4 生效") - run.add_argument("--rl-model", default=None, - help="RL 模型路径（默认 output/rl/q3_ppo.zip 或 " - "output/rl/q4_ppo.zip；训练脚本 scripts/train_rl.py）") - run.add_argument("--hrl-model", default=None, - help="高层 HRL MaskablePPO 模型路径（--strategy hrl 时必填；" - "训练脚本 scripts/train_hrl_high.py）") - run.add_argument("--hrl-low-model", default=None, - help="可选低层 HRL 模型；当前只用于轨迹推演留痕，不改变协议终点") - run.add_argument("--hrl-device", choices=("auto", "cpu", "cuda", "mps"), - default="auto", help="HRL 模型推理设备") - run.add_argument("--trace-low", action="store_true", - help="开启低层到达推演留痕（本地虚拟轨迹，不增加协议耗时）") + "grid=49点网格基线；仅 --problem 3 生效") run.add_argument("--bearing-margin-deg", type=float, default=0.005, help="示向度舍入裕量（服务器保留两位小数；默认 0.005°）") run.add_argument("--real-budget-s", type=float, default=None, diff --git a/src/mcm26b/hrl/__init__.py b/src/mcm26b/hrl/__init__.py deleted file mode 100644 index 4f33090..0000000 --- a/src/mcm26b/hrl/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -"""分层强化学习实验：整场子目标调度环境与独立的八方向到达实验。 - -与 :mod:`mcm26b.rl`（单步 PPO 选点）的差异：高层在整场 10–16 源的统一 -候选槽位上做调度决策（证书清除 / 信息增益定位 / 覆盖站扫频 / 后备清除）， -当前运行闭环将子目标端点提交协议，八方向策略仅用于可选本地路径留痕， -不影响真实请求及虚拟计时。完整高层训练、CLI 与配对评估仍待接通； -优化架构及逐项验收见 docs/hrl-sol-execution.md。 - -信念（belief + 网格幸存集）、证书审批与后备清除复用 planner 基础设施； -每次改动仍须验证真源保留和真实全清，不能用内部状态自洽替代评测。 - -本包属于训练/推理侧组件，运行时对依赖保持惰性（stable-baselines3 / -sb3-contrib 仅在显式加载模型时导入）。 -""" diff --git a/src/mcm26b/hrl/constants.py b/src/mcm26b/hrl/constants.py deleted file mode 100644 index 17704ad..0000000 --- a/src/mcm26b/hrl/constants.py +++ /dev/null @@ -1,49 +0,0 @@ -"""HRL 观测/动作空间常量：模型接口的唯一字面量来源。 - -槽位布局（训练与推理共享，改动任一值都会使已训练模型失配，须同步重训）： - - 0–19 clear 频道 c 的证书清除（槽位 = c-1，证书就绪才有效） - 20–39 locate 频道 c 的定位测量（信息增益选点或定向遮挡梯子探点） - 40–88 station 覆盖站（问题3：七点环 0–6 有效；问题4：49 点网格） - 89–108 fallback 频道 c 的后备网格清除（仅问题4，遮挡梯子耗尽后开放） - 109–119 保留（恒屏蔽） - -高层观测 = 全局 8 维 + 20 频道 × 14 维 + 120 槽位 × 8 维 = 1248 维； -低层观测 10 维、动作 Discrete(8)（8 方向，步长 ≤100 m）。 -""" - -FEATURE_VERSION = 4 -ACTION_LAYOUT_VERSION = 1 -REWARD_VERSION = 1 -CHANNEL_COUNT = 20 - -# ------------------------------------------------------------ 高层动作槽位 -CLEAR_SLOTS = CHANNEL_COUNT -LOCATE_SLOTS = CHANNEL_COUNT -STATION_SLOTS = 49 -FALLBACK_SLOTS = CHANNEL_COUNT -CLEAR_OFFSET = 0 -LOCATE_OFFSET = CLEAR_OFFSET + CLEAR_SLOTS -STATION_OFFSET = LOCATE_OFFSET + LOCATE_SLOTS -FALLBACK_OFFSET = STATION_OFFSET + STATION_SLOTS -N_CANDIDATES = 120 - -# ------------------------------------------------------------ 高层观测 -PER_CHANNEL_DIM = 14 -HIGH_GLOBAL_DIM = 8 -PER_CANDIDATE_DIM = 8 -HIGH_OBS_DIM = (HIGH_GLOBAL_DIM + CHANNEL_COUNT * PER_CHANNEL_DIM - + N_CANDIDATES * PER_CANDIDATE_DIM) # 1248 - -# ------------------------------------------------------------ 低层接口 -LOW_OBS_DIM = 10 -LOW_NUM_ACTIONS = 8 # 8 方向，45° 等分 -LOW_STEP_M = 100.0 # 每步最大步长 [题面 低层动作 50–100 m] -LOW_REACH_M = 20.0 # 到达阈值（与清除半径一致） -LOW_MAX_STEPS = 60 # 直径穿越 ~51 步上界 - -# ------------------------------------------------------------ 观测归一化 -POS_SCALE_M = 2000.0 # 位置/位移归一化尺度（Ω 半径 1800 m 外扩） -RADIUS_SCALE_M = 2000.0 -COST_SCALE_S = 600.0 # 槽位预计服务耗时归一化尺度 -MEASURE_COUNT_SCALE = 12.0 diff --git a/src/mcm26b/hrl/curriculum.py b/src/mcm26b/hrl/curriculum.py deleted file mode 100644 index 6649793..0000000 --- a/src/mcm26b/hrl/curriculum.py +++ /dev/null @@ -1,109 +0,0 @@ -"""课程学习与域随机化：训练场景按阶段由易到难（prompt §7）。 - -四阶段课程：1 源/5 频道/无噪声 → 3 源/10 频道/±0.5° → 8 源/16 频道/±1° -→ 10–16 源/20 频道/±1°（问题4 最后阶段按比例混入定向源）。域随机化 -覆盖源位置（Ω 圆域均匀）、频道（互异抽样）、有效接收半径、朝向与逐局 -误差种子 [prompt §7.2]；机器狗初始位置固定 (0,0)——协议语义如此， -随机化它反而制造训练-部署差异（与 v1"训练部署特征错位"教训一致）。 - -真值只进入训练/评估场景生成，不进入任何观测；策略端仅见模拟器响应。 -""" - -from __future__ import annotations - -import dataclasses -import math -from dataclasses import dataclass -from random import Random - -from ..config import ProblemConfig - - -@dataclass(frozen=True) -class Stage: - """课程阶段参数：源数范围、频道上限、示向度噪声与定向源比例。""" - - name: str - sources_min: int - sources_max: int - channel_max: int # 频道 > channel_max 的频道保证无源 - bearing_error_half_deg: float - directional_ratio: float # 问题4 最后阶段定向源比例；问题3 恒 0 - recommended_timesteps: int - - -STAGES: tuple[Stage, ...] = ( - Stage("s1-single", 1, 1, 5, 0.0, 0.0, 60_000), - Stage("s2-few", 3, 3, 10, 0.5, 0.0, 80_000), - Stage("s3-many", 8, 8, 16, 1.0, 0.0, 100_000), - Stage("s4-full", 10, 16, 20, 1.0, 0.4, 140_000), -) - - -def stage_with_noise(config: ProblemConfig, stage: Stage) -> ProblemConfig: - """把阶段噪声写入 ProblemConfig（belief/PF 的保守裕量随之同步）。""" - field = dataclasses.replace( - config.field, bearing_error_half_deg=stage.bearing_error_half_deg) - return dataclasses.replace(config, field=field) - - -# 快速模式只减少选点评分的样本，不改变证明使用的幸存网格。29.3m 格距的 -# Voronoi 裕量约20.72m，已经超过20m清除半径，会使粒子证书永远不可达。 -# 默认使用生产配置；快速模式必须显式指定并在训练元数据记录。 -TRAINING_SCORE_SUBSAMPLE = 240 -TRAINING_MC_REPRESENTATIVES = 5 - - -def training_config(config: ProblemConfig, stage: Stage, *, - fidelity: str = "production") -> ProblemConfig: - """阶段噪声配置；fast只降低评分采样量，production与部署同参数。""" - if fidelity not in ("production", "fast"): - raise ValueError("fidelity 必须是 production 或 fast") - noisy = stage_with_noise(config, stage) - if fidelity == "production": - return noisy - bayes = dataclasses.replace( - noisy.bayes, - score_subsample=TRAINING_SCORE_SUBSAMPLE, - mc_representatives=TRAINING_MC_REPRESENTATIVES) - return dataclasses.replace(noisy, bayes=bayes) - - -def make_scene(rng: Random, config: ProblemConfig, problem: int, - stage: Stage, *, domain_randomization: bool = True - ) -> tuple[list, list[int]]: - """生成一局课程场景，返回 (referee.Source 列表, 已占用频道列表)。 - - 源位置在 Ω 圆域均匀分布、频道从 1..stage.channel_max 互异抽样、 - 有效接收半径 [1000,1500] 均匀 [题面]；问题4 按 stage.directional_ratio - 混入定向源，朝向均匀。domain_randomization=False 时只用 rng 的确定 - 序列（仍随机但可整场复现），供评估固定场景对照。 - """ - from referee import Source - - field = config.field - count = rng.randint(stage.sources_min, stage.sources_max) - channels = rng.sample(range(field.channel_min, stage.channel_max + 1), count) - sources = [] - for channel in channels: - radius = field.omega_radius_m * math.sqrt(rng.random()) - angle = rng.uniform(0.0, 2.0 * math.pi) - r_eff = rng.uniform(field.r_eff_min_m, field.r_eff_max_m) - directional = (problem == 4 - and rng.random() < stage.directional_ratio) - heading = rng.uniform(0.0, 360.0) if directional else 0.0 - if not domain_randomization: - # 固定化退耦：半径/朝向取区间代表值，保留位置随机（评估用）。 - r_eff = field.r_eff_min_m - sources.append(Source(channel=channel, x=radius * math.cos(angle), - y=radius * math.sin(angle), r_eff_m=r_eff, - directional=directional, heading_deg=heading)) - return sources, channels - - -def training_spec(config: ProblemConfig, stage: Stage, *, - max_real_duration_s: float = 3600.0): - """带阶段噪声的 referee Spec（训练侧使用；正式对接以 /enter 为准）。""" - from ..harness import referee_spec - return referee_spec(stage_with_noise(config, stage), - max_real_duration_s=max_real_duration_s) diff --git a/src/mcm26b/hrl/executor.py b/src/mcm26b/hrl/executor.py deleted file mode 100644 index 96f6dbe..0000000 --- a/src/mcm26b/hrl/executor.py +++ /dev/null @@ -1,113 +0,0 @@ -"""低层到达执行器与观测：8 方向步进（prompt §4.2），训练/部署同源。 - -低层只负责"怎么去"：给定当前位置与子目标，输出离散 8 方向步进序列。 -步长每步 ≤ ``LOW_STEP_M``，近目标时缩短步长；到达阈值 ``LOW_REACH_M`` -与清除半径一致。推演完全本地，不与模拟器交互、不产生虚拟耗时——协议 -语义下移动耗时 = 两次动作位置间的直线距离 ÷ 速度，仅由终点决定。 - -本模块仅供独立导航实验和轨迹诊断；当前 HTTP 执行路径仍直接服务高层 -目标，不能据此声称低层 PPO 改善正式任务耗时。policy=None 使用解析 -直线控制器；PPO 推演超时不会免费跳到目标。 -""" - -from __future__ import annotations - -import math - -import numpy as np - -from ..config import Vec2 -from .constants import (LOW_MAX_STEPS, LOW_NUM_ACTIONS, - LOW_REACH_M, LOW_STEP_M, POS_SCALE_M) - -__all__ = ["build_low_observation", "LowLevelExecutor", "action_bearing", - "navigation_transition"] - - -def action_bearing(action: int) -> float: - """离散方向 -> 弧度方位（0 = +x，逆时针 45° 等分）。""" - return float(action) * 2.0 * math.pi / LOW_NUM_ACTIONS - - -def navigation_transition(position: Vec2, goal: Vec2, action: int, *, - step_m: float = LOW_STEP_M) -> tuple[Vec2, float]: - """训练与轨迹推演共享的运动学；返回位置和实际位移长度。 - - 方向仍由策略决定，近目标时仅缩短步长，不把位置吸附到目标。 - 这是无障碍平面导航实验；不引入观测不可见的绝对位置裁剪。 - """ - if not 0 <= int(action) < LOW_NUM_ACTIONS: - raise ValueError("低层动作必须是 0..7") - if not math.isfinite(step_m) or step_m <= 0: - raise ValueError("step_m 必须是有限正数") - distance = min(float(step_m), math.dist(position, goal)) - bearing = action_bearing(int(action)) - next_position = (position[0] + distance * math.cos(bearing), - position[1] + distance * math.sin(bearing)) - return next_position, math.dist(position, next_position) - - -def build_low_observation(position: Vec2, goal: Vec2, prev_action: int | None, - steps: int) -> np.ndarray: - """低层观测（shape=(LOW_OBS_DIM,)，训练/推理逐位一致）。""" - dx, dy = goal[0] - position[0], goal[1] - position[1] - distance = math.hypot(dx, dy) - bearing = math.atan2(dy, dx) - prev_sin = prev_cos = 0.0 - if prev_action is not None: - prev_bearing = action_bearing(prev_action) - prev_sin, prev_cos = math.sin(prev_bearing), math.cos(prev_bearing) - return np.asarray([ - dx / POS_SCALE_M, - dy / POS_SCALE_M, - distance / POS_SCALE_M, - math.sin(bearing), - math.cos(bearing), - steps / LOW_MAX_STEPS, - prev_sin, - prev_cos, - min(distance / LOW_STEP_M, 3.0) / 3.0, - 1.0, - ], dtype=np.float32) - - -class LowLevelExecutor: - """子目标到达推演：低层 PPO 或直线控制器（policy=None）。""" - - def __init__(self, policy=None, *, step_m: float = LOW_STEP_M, - reach_m: float = LOW_REACH_M, max_steps: int = LOW_MAX_STEPS): - self.policy = policy - self.step_m = float(step_m) - self.reach_m = float(reach_m) - self.max_steps = int(max_steps) - self.last_roll_info: dict = {} - - def roll(self, start: Vec2, goal: Vec2) -> list[Vec2]: - """返回真实推演途经点；last_roll_info 如实记录到达/截断。""" - goal = (float(goal[0]), float(goal[1])) - if self.policy is None: - self.last_roll_info = {"reached": True, "truncated": False, - "final_distance_m": 0.0, - "path_length_m": math.dist(start, goal)} - return [goal] - position = (float(start[0]), float(start[1])) - waypoints: list[Vec2] = [] - prev_action: int | None = None - path_length = 0.0 - for steps in range(self.max_steps): - if math.dist(position, goal) <= self.reach_m: - break - observation = build_low_observation(position, goal, prev_action, - steps) - action = int(self.policy.select(observation)) - position, moved = navigation_transition(position, goal, action, - step_m=self.step_m) - path_length += moved - waypoints.append(position) - prev_action = action - distance = math.dist(position, goal) - self.last_roll_info = {"reached": distance <= self.reach_m, - "truncated": distance > self.reach_m, - "final_distance_m": distance, - "path_length_m": path_length} - return waypoints diff --git a/src/mcm26b/hrl/hierarchical_env.py b/src/mcm26b/hrl/hierarchical_env.py deleted file mode 100644 index 92d35bb..0000000 --- a/src/mcm26b/hrl/hierarchical_env.py +++ /dev/null @@ -1,260 +0,0 @@ -"""高层 PPO 训练环境：整场 10–16 源的发现-定位-清除调度（prompt §2/§5.2）。 - -与 v2 :class:`~mcm26b.rl.env.LocalizationEnv`（单源 ≤16 步子任务）的本质 -差异：本环境的每一步是一次完整的高层调度决策（候选槽位选择 + 任务执行）， -环境语义与运行闭环 :class:`~mcm26b.planner.hrl_loop.HRLLoop` 共用同一 -混入层（hrl.scheduler），训练-部署观测逐位同源。 - -奖励（prompt §5.2 的可折算实现，权重见 :class:`HRLReward`）： - r = -Δ虚拟耗时/20 + 100·清除 + 5·新发现 + 0.5·Σ熵减 - 2·冗余检测 -终止：全部频道清除/证明缺席（+50）；截断仍有未决频道（-100）；异常（-100）。 -主口径是虚拟耗时——与题目目标函数（最小化平均定位清除时间/虚拟总时间） -一致，清除/发现/熵减是塑形项（消融实验逐项验证其必要性）。 - -真值只在 reset 布景时使用，绝不进入观测；感知读数全部来自 -referee.LocalSimulator（与官方 HTTP 模拟器同协议语义）。 -""" - -from __future__ import annotations - -from dataclasses import dataclass -from random import Random - -import gymnasium as gym -import numpy as np -from gymnasium import spaces - -from ..belief import ChannelStatus -from ..config import ProblemConfig -from ..hrl.curriculum import STAGES, Stage, make_scene, training_config -from ..hrl.constants import HIGH_OBS_DIM, N_CANDIDATES -from ..planner.hrl_loop import HRLLoop -from ..protocol.client import SimulatorClient -from ..protocol.log import SessionLog - -__all__ = ["HRLReward", "HierarchicalEnv"] - - -@dataclass -class HRLReward: - """高层奖励权重。 - - 默认保留 prompt §5.2 的可解释塑形，便于对照实验；P0.4 建议的 - 纯时间口径可通过 :meth:`time_only` 显式启用，训练脚本默认使用它。 - """ - - time_scale: float = 20.0 # r -= Δ虚拟耗时 / time_scale - clear: float = 100.0 # 每清除一个源 - detect: float = 5.0 # 每新发现一个频道有信号 - info_weight: float = 0.5 # 粒子云熵减（log 幸存格数）内在奖励 - redundant: float = 2.0 # 同频道同点重复检测惩罚 - invalid: float = 2.0 # 掩码外动作（无掩码消融路径） - terminal_accounted: float = 50.0 # 全部频道清除/证毕的收尾奖励 - terminal_unresolved: float = 100.0 # 截断/停机时仍有未决频道的惩罚 - failure: float = 100.0 # 协议/守恒性异常惩罚 - - @classmethod - def time_only(cls) -> "HRLReward": - """P0.4 纯时间目标：只惩罚已发生虚拟时间，去掉塑形项。 - - 终止奖/罚是固定常数，不逐源叠加；失败罚按虚拟时限/时间尺度 - 设定为足以覆盖一整局时间成本。 - """ - return cls( - time_scale=100.0, - clear=0.0, - detect=0.0, - info_weight=0.0, - redundant=0.0, - invalid=2.0, - terminal_accounted=30.0, - terminal_unresolved=300.0, - failure=3600.0, - ) - - -class HierarchicalEnv(gym.Env): - """整场分层调度 Gymnasium 环境（问题3/4，课程参数化）。 - - stage=None 时使用 STAGES 末阶段（完整任务口径）；课程训练按阶段 - 传入。domain_randomization=False 保留随机位置但固定半径代表值， - 供配对评估。 - """ - - metadata = {"render_modes": []} - - def __init__(self, config: ProblemConfig | None = None, *, - problem: int = 3, stage: Stage | None = None, - max_episode_steps: int = 300, - domain_randomization: bool = True, - reward: HRLReward | None = None, - real_budget_s: float = 3600.0, - error_seed: int | None = None, - fidelity: str = "production"): - super().__init__() - if problem not in (3, 4): - raise ValueError("problem 必须是 3 或 4") - self.base_config = config or ProblemConfig() - self.problem = int(problem) - self.stage = stage or STAGES[-1] - self.max_episode_steps = int(max_episode_steps) - self.domain_randomization = bool(domain_randomization) - self.reward = reward or HRLReward() - self.real_budget_s = float(real_budget_s) - self.error_seed = error_seed - self.fidelity = fidelity - # 默认与部署同精度；fast仅减少评分采样，保留证书网格精度。 - self.train_config = training_config(self.base_config, self.stage, - fidelity=fidelity) - self.observation_space = spaces.Box(low=-12.0, high=12.0, - shape=(HIGH_OBS_DIM,), - dtype=np.float32) - self.action_space = spaces.Discrete(N_CANDIDATES) - - self._scene_rng = Random() - self._loop: HRLLoop | None = None - self._client: SimulatorClient | None = None - self._steps = 0 - self._terminated = False - self._truncated = False - self._start_virtual_s = 0.0 - self._current_candidates: list = [] - self._failure_reason: str | None = None - self._last_outcome = None - - # ------------------------------------------------------------ 场景 - def _make_episode(self) -> None: - from ..harness import LocalSimTransport - from referee import LocalSimulator - - sources, _channels = make_scene( - self._scene_rng, self.train_config, self.problem, self.stage, - domain_randomization=self.domain_randomization) - self._simulator = LocalSimulator( - sources, robot_id="hrl-train", - spec=self._training_spec(), - error_seed=(self.error_seed if self.error_seed is not None - else self._scene_rng.randint(0, 2 ** 31 - 1)), - stress_mode=True) - self._client = SimulatorClient( - "hrl-train", config=self.train_config, - transport=LocalSimTransport(self._simulator), log=SessionLog()) - self._client.enter() - self._loop = HRLLoop(self._client, self.train_config, - problem=self.problem) - - def _training_spec(self): - from ..hrl.curriculum import training_spec - return training_spec(self.train_config, self.stage, - max_real_duration_s=self.real_budget_s) - - # ------------------------------------------------------------ Gym API - def reset(self, *, seed: int | None = None, options: dict | None = None): - super().reset(seed=seed) - if seed is not None: - self._scene_rng = Random(seed) - self._steps = 0 - self._terminated = False - self._truncated = False - self._failure_reason = None - self._make_episode() - self._start_virtual_s = self._client.state.virtual_time_s - self._current_candidates = self._loop.build_candidates() - observation = self._loop.hrl_observation(self._current_candidates) - return observation, self._info() - - def action_masks(self) -> np.ndarray: - """MaskablePPO 动作掩码接口（与观测同源的候选槽位有效性）。""" - return self._loop.hrl_action_mask(self._current_candidates) - - def _info(self) -> dict: - loop = self._loop - belief = loop.belief - cleared = len(loop.cleared) - absent = sum(1 for ch in belief.channels.values() - if ch.status is ChannelStatus.ABSENT) - return { - "steps": self._steps, - "cleared": cleared, - "absent_proven": absent, - "accounted": cleared + absent, - "virtual_time_s": self._client.state.virtual_time_s, - "episode_virtual_cost_s": (self._client.state.virtual_time_s - - self._start_virtual_s), - "stop_reason": loop.stop_reason, - "failure_reason": self._failure_reason, - "anomalies": list(loop.anomalies), - } - - def step(self, action): - if self._terminated or self._truncated: - raise RuntimeError("step() 需要在 terminated/truncated 后先 reset()") - loop = self._loop - self._steps += 1 - try: - outcome = loop.hrl_step(action, candidates=self._current_candidates) - except Exception as exc: # 协议/守恒性异常：与 v2 环境同口径收尾 - self._failure_reason = type(exc).__name__ + ': ' + str(exc) - observation = np.zeros((HIGH_OBS_DIM,), dtype=np.float32) - self._terminated = True - return (observation, -self.reward.failure, True, False, - self._info()) - self._last_outcome = outcome - - if outcome.invalid: - reward = -self.reward.invalid - else: - reward = self._reward_from(outcome) - - # 终止判定：停机原因/全部证毕；或候选耗尽时走运行闭环 run() 的 - # 同款安全网（覆盖缺席标记 + 阻塞定位清除），安全网的虚拟耗时 - # 计入本步奖励（归因给促成该局面的决策序列）。 - terminated = loop.stop_reason is not None or loop._all_accounted() - if not terminated: - self._current_candidates = loop.build_candidates() - if not self._current_candidates: - virtual_before = self._client.state.virtual_time_s - loop._mark_absent_after_coverage() - loop._resolve_detected() - reward -= (self._client.state.virtual_time_s - virtual_before) \ - / self.reward.time_scale - self._current_candidates = [] - terminated = True - else: - self._current_candidates = [] - - if terminated: - reward += (self.reward.terminal_accounted - if loop._all_accounted() - else -self.reward.terminal_unresolved) - truncated = False - if not terminated and self._steps >= self.max_episode_steps: - truncated = True - if not loop._all_accounted(): - reward -= self.reward.terminal_unresolved - self._terminated = terminated - self._truncated = truncated - if terminated or truncated: - observation = np.zeros((HIGH_OBS_DIM,), dtype=np.float32) - else: - self._current_candidates = loop.build_candidates() - observation = loop.hrl_observation(self._current_candidates) - return observation, float(reward), terminated, truncated, self._info() - - def _reward_from(self, outcome) -> float: - reward = -outcome.delta_virtual_s / self.reward.time_scale - reward += self.reward.clear * len(outcome.clear_success) - reward += self.reward.detect * len(outcome.new_detections) - reward += self.reward.info_weight * sum(outcome.info_gains.values()) - if outcome.redundant: - reward -= self.reward.redundant - return reward - - def close(self): - client = getattr(self, "_client", None) - if client is not None and client.state.entered \ - and not client.state.exited: - try: - client.exit() - except Exception: - pass diff --git a/src/mcm26b/hrl/low_level_env.py b/src/mcm26b/hrl/low_level_env.py deleted file mode 100644 index c8788da..0000000 --- a/src/mcm26b/hrl/low_level_env.py +++ /dev/null @@ -1,158 +0,0 @@ -"""低层 PPO 训练环境：随机起点-子目标的 8 方向到达任务（prompt §4.2/§5.1）。 - -奖励为实际位移耗时惩罚、转向惩罚与到达奖励。步长上限 100m，近目标 -缩短步长，避免固定步长振荡。此环境仅用于独立导航实验；正式 HTTP -路径尚不执行低层 PPO 的途经点，不能将导航成绩当成正式任务收益。 - -策略端只见相对位姿观测，真值（子目标坐标）经相对向量进入观测， -与部署执行器完全同源（build_low_observation）。 -""" - -from __future__ import annotations - -import math -from random import Random - -import gymnasium as gym -import numpy as np -from gymnasium import spaces - -from ..config import ProblemConfig -from .constants import (LOW_MAX_STEPS, LOW_NUM_ACTIONS, LOW_OBS_DIM, - LOW_REACH_M, LOW_STEP_M) -from .executor import (action_bearing, build_low_observation, - navigation_transition) - -__all__ = ["LowLevelEnv"] - - -class LowLevelEnv(gym.Env): - """低层到达任务：观测 10 维，动作 Discrete(8)，逐步计费。""" - - metadata = {"render_modes": []} - - def __init__(self, config: ProblemConfig | None = None, *, - seed: int | None = None, - step_m: float = LOW_STEP_M, reach_m: float = LOW_REACH_M, - max_steps: int = LOW_MAX_STEPS, - min_distance_m: float = 300.0, - turn_penalty: float = 0.05, reach_bonus: float = 2.0, - step_cost_scale: float = 100.0, - truncation_penalty: float = 2.0): - super().__init__() - self.config = config or ProblemConfig() - self.step_m = float(step_m) - self.reach_m = float(reach_m) - self.max_steps = int(max_steps) - self.min_distance_m = float(min_distance_m) - self.turn_penalty = float(turn_penalty) - self.reach_bonus = float(reach_bonus) - self.step_cost_scale = float(step_cost_scale) - self.truncation_penalty = float(truncation_penalty) - self.observation_space = spaces.Box(low=-np.inf, high=np.inf, - shape=(LOW_OBS_DIM,), - dtype=np.float32) - self.action_space = spaces.Discrete(LOW_NUM_ACTIONS) - self._rng = Random(seed) - self._position = (0.0, 0.0) - self._goal = (0.0, 0.0) - self._steps = 0 - self._prev_action: int | None = None - self._terminated = False - self._truncated = False - self._path_length_m = 0.0 - self._virtual_time_s = 0.0 - self._straight_m = 0.0 - - # ------------------------------------------------------------ Gym API - def reset(self, *, seed: int | None = None, options: dict | None = None): - super().reset(seed=seed) - if seed is not None: - self._rng = Random(seed) - radius = self.config.field.omega_radius_m - self._position = self._sample_point(radius) - for _ in range(100): - self._goal = self._sample_point(radius) - if math.dist(self._position, self._goal) >= self.min_distance_m: - break - self._steps = 0 - self._prev_action = None - self._terminated = False - self._truncated = False - self._path_length_m = 0.0 - self._virtual_time_s = 0.0 - self._straight_m = math.dist(self._position, self._goal) - return self._observe(), self._info() - - def action_masks(self) -> np.ndarray: - """无障碍低层八个方向均可用，兼容 MaskablePPO。""" - return np.ones(LOW_NUM_ACTIONS, dtype=bool) - - def _sample_point(self, radius: float) -> tuple[float, float]: - angle = self._rng.uniform(0.0, 2.0 * math.pi) - distance = radius * math.sqrt(self._rng.random()) - return (distance * math.cos(angle), distance * math.sin(angle)) - - def step(self, action): - if self._terminated or self._truncated: - raise RuntimeError("step() 需要在 terminated/truncated 后先 reset()") - action = int(action) - if self._prev_action is not None: - turn = abs((action_bearing(action) - - action_bearing(self._prev_action) + math.pi) - % (2.0 * math.pi) - math.pi) / math.pi - else: - turn = 0.0 - self._position, moved = navigation_transition( - self._position, self._goal, action, step_m=self.step_m) - self._prev_action = action - self._steps += 1 - - step_cost_s = moved / self.config.cost.speed_mps - self._path_length_m += moved - self._virtual_time_s += step_cost_s - reward = -step_cost_s / self.step_cost_scale - reward -= self.turn_penalty * turn - distance = math.dist(self._position, self._goal) - terminated = distance <= self.reach_m - truncated = False - if terminated: - reward += self.reach_bonus - elif self._steps >= self.max_steps: - truncated = True - reward -= self.truncation_penalty - self._terminated = terminated - self._truncated = truncated - info = self._info() - info["final_distance_m"] = distance - info["step_distance_m"] = moved - info["step_virtual_time_s"] = step_cost_s - return self._observe(), float(reward), terminated, truncated, info - - def _observe(self) -> np.ndarray: - return build_low_observation(self._position, self._goal, - self._prev_action, self._steps) - - def _info(self) -> dict: - return { - "position": self._position, - "goal": self._goal, - "steps": self._steps, - "distance_m": math.dist(self._position, self._goal), - "straight_m": self._straight_m, - "path_length_m": self._path_length_m, - "virtual_time_s": self._virtual_time_s, - } - - # ------------------------------------------------------------ 专家（BC） - def expert_action(self) -> int: - """直线专家：选择方位最接近子目标方向的离散方向（BC 标签）。""" - bearing = math.atan2(self._goal[1] - self._position[1], - self._goal[0] - self._position[0]) - best, best_gap = 0, float("inf") - for action in range(LOW_NUM_ACTIONS): - gap = abs((action_bearing(action) - bearing + math.pi) - % (2.0 * math.pi) - math.pi) - if gap < best_gap: - best, best_gap = action, gap - return best diff --git a/src/mcm26b/hrl/policy.py b/src/mcm26b/hrl/policy.py deleted file mode 100644 index 4b71cbd..0000000 --- a/src/mcm26b/hrl/policy.py +++ /dev/null @@ -1,135 +0,0 @@ -"""HRL 策略加载与推理（运行时接入 planner 的唯一入口，模式同 rl.policy）。 - -模型文件由 scripts/train_hrl_low.py / train_hrl_high.py 生成，并伴随同名 -JSON 元数据（feature_version/algorithm/problem/维度）；加载时校验，失配 -即拒绝运行，杜绝静默错位。stable-baselines3 / torch 均为惰性导入。 -""" - -from __future__ import annotations - -import json -from pathlib import Path - -import numpy as np - -from .constants import (ACTION_LAYOUT_VERSION, FEATURE_VERSION, - HIGH_OBS_DIM, LOW_NUM_ACTIONS, LOW_OBS_DIM, - N_CANDIDATES, REWARD_VERSION) - -__all__ = ["HighLevelPolicy", "LowLevelPolicy", - "default_high_model_path", "default_low_model_path"] - - -def _resolve_device(device: str) -> str: - if device != "auto": - return device - try: - import torch - return "cuda" if torch.cuda.is_available() else "cpu" - except ImportError: - return "cpu" - - -def _load_maskable(model_path: Path, device: str, *, - obs_dim: int, act_dim: int, problem: int): - if not model_path.is_file(): - raise RuntimeError( - f"未找到 HRL 模型 {model_path}。请先训练：\n" - f" uv sync --group rl --group dev\n" - f" uv run python scripts/train_hrl_high.py --problem {problem}") - try: - from sb3_contrib import MaskablePPO - except ImportError as exc: - raise RuntimeError( - "缺少 RL 运行依赖（sb3-contrib）。请执行 `uv sync --group rl` " - "后重试。") from exc - meta_path = model_path.with_suffix('.json') - metadata = json.loads(meta_path.read_text(encoding='utf-8')) \ - if meta_path.exists() else {} - # 高层模型必须携带完整语义元数据；缺字段不能默认兼容，防止旧槽位/ - # 旧奖励语义模型被静默加载。低层导航模型只校验基础特征版本。 - strict = act_dim == N_CANDIDATES - if strict: - required = ("feature_version", "action_layout_version", - "reward_version", "algorithm", "problem", - "obs_dim", "act_dim") - missing = [key for key in required if key not in metadata] - if missing: - raise RuntimeError( - f"HRL 高层模型元数据缺少字段 {missing}；请用当前 " - f"scripts/train_hrl_high.py 重训。") - if metadata.get("feature_version") != FEATURE_VERSION: - raise RuntimeError( - f"HRL 模型特征版本失配: 期望 {FEATURE_VERSION}，" - f"实际 {metadata.get('feature_version')}。请用当前版本重训。") - if metadata.get("action_layout_version") != ACTION_LAYOUT_VERSION: - raise RuntimeError( - f"HRL 模型动作布局版本失配: 期望 {ACTION_LAYOUT_VERSION}，" - f"实际 {metadata.get('action_layout_version')}。请重训。") - if metadata.get("reward_version") != REWARD_VERSION: - raise RuntimeError( - f"HRL 模型奖励版本失配: 期望 {REWARD_VERSION}，" - f"实际 {metadata.get('reward_version')}。请重训。") - else: - if metadata.get("feature_version") != FEATURE_VERSION: - raise RuntimeError( - f"低层模型特征版本失配: 期望 {FEATURE_VERSION}，" - f"实际 {metadata.get('feature_version')}。请重训。") - if metadata.get("algorithm") != "maskable_ppo": - raise RuntimeError("HRL 模型必须是 maskable_ppo 训练产物") - if problem is not None and metadata.get("problem") not in (None, problem): - raise RuntimeError("模型训练问题编号与当前第三/四问不匹配") - model = MaskablePPO.load(str(model_path), device=device) - try: - loaded_obs = model.observation_space.shape[0] - loaded_act = model.action_space.n - except Exception as exc: - raise RuntimeError(f"模型空间与 HRL 接口不兼容: {model_path}") from exc - if loaded_obs != obs_dim or loaded_act != act_dim: - raise RuntimeError( - f"模型接口失配: 期望 obs={obs_dim}/act={act_dim}，" - f"实际 obs={loaded_obs}/act={loaded_act}。请重训。") - return model - - -class HighLevelPolicy: - """高层子目标调度策略：观测+掩码 -> 槽位编号（确定性 argmax）。""" - - def __init__(self, model_path: str | Path, *, device: str = "auto", - problem: int | None = None): - self.device = _resolve_device(device) - self.path = Path(model_path) - self.model = _load_maskable(self.path, self.device, obs_dim=HIGH_OBS_DIM, - act_dim=N_CANDIDATES, problem=problem) - meta_path = self.path.with_suffix('.json') - self.metadata = json.loads(meta_path.read_text(encoding='utf-8')) \ - if meta_path.exists() else {} - - def select(self, observation: np.ndarray, action_masks: np.ndarray) -> int: - action, _state = self.model.predict( - observation, action_masks=action_masks, deterministic=True) - return int(action) - - -class LowLevelPolicy: - """低层到达策略：相对位姿观测 -> 8 方向（确定性 argmax）。""" - - def __init__(self, model_path: str | Path, *, device: str = "auto"): - self.device = _resolve_device(device) - self.path = Path(model_path) - self.model = _load_maskable(self.path, self.device, obs_dim=LOW_OBS_DIM, - act_dim=LOW_NUM_ACTIONS, problem=None) - - def select(self, observation: np.ndarray) -> int: - action, _state = self.model.predict(observation, deterministic=True) - return int(action) - - -def default_high_model_path(problem: int) -> Path: - if problem not in (3, 4): - raise ValueError("problem 必须是 3 或 4") - return Path("output/hrl") / f"q{problem}_hrl_high.zip" - - -def default_low_model_path() -> Path: - return Path("output/hrl") / "hrl_low.zip" diff --git a/src/mcm26b/hrl/scheduler.py b/src/mcm26b/hrl/scheduler.py deleted file mode 100644 index d03a081..0000000 --- a/src/mcm26b/hrl/scheduler.py +++ /dev/null @@ -1,495 +0,0 @@ -"""高层调度核心：候选槽位构建、观测、动作掩码与任务执行。 - -本模块是 :class:`~mcm26b.planner.hrl_loop.HRLLoop`（运行闭环）与 -:class:`~mcm26b.hrl.hierarchical_env.HierarchicalEnv`（训练环境）共用的 -混入层，保证训练-部署特征逐位同源（v1"训练部署特征错位"审核教训）。 - -宿主须为 :class:`~mcm26b.planner.bayes_loop.BayesLoop` 派生闭环，提供 -client/belief/particles/ledger/_record_measure 等感知与执行原语；本层只做 -三件事—— - -1. 把整场调度决策离散化为固定槽位（constants.N_CANDIDATES）： - clear(20) + locate(20) + station(49) + fallback(20) + 保留(11)； -2. 由可见观测（belief + 网格幸存集 + 覆盖账本）构造观测与动作掩码， - 绝不读取源真值； -3. 执行选中候选并报告事实（虚拟耗时增量、新发现、熵减、清除结果）， - 奖励折算由训练环境负责。 - -Q4 定向源语义：no_signal 不能删除空间候选（可能是朝向遮挡），故问题4 的 -幸存集只按 direction/near 更新；遮挡证据走 blocked_probes 梯子与后备清除 -槽位，与 DirectionalExtension 的保守口径一致。 -""" - -from __future__ import annotations - -import math -from dataclasses import dataclass, field -from typing import NamedTuple - -import numpy as np - -from ..belief import ChannelStatus -from ..config import Vec2 -from ..protocol.rules import CLEAR_SUCCESS, DIRECTION, NEAR, NO_SIGNAL -from ..protocol.timing import move_time -from .constants import (CHANNEL_COUNT, CLEAR_OFFSET, COST_SCALE_S, - FALLBACK_OFFSET, HIGH_GLOBAL_DIM, HIGH_OBS_DIM, - LOCATE_OFFSET, MEASURE_COUNT_SCALE, N_CANDIDATES, - PER_CANDIDATE_DIM, PER_CHANNEL_DIM, POS_SCALE_M, - RADIUS_SCALE_M, STATION_OFFSET) -from ..planner import fallback_clear_cells, safe_disk_radius - -__all__ = ["Candidate", "StepOutcome", "HRLControllerMixin", "point_key"] - - -def point_key(point: Vec2) -> tuple[int, int]: - """候选点去重键：0.5m 网格取整（与 bayes_loop 同口径）。""" - return (int(round(point[0] * 2)), int(round(point[1] * 2))) - - -class Candidate(NamedTuple): - """一个高层候选子目标。slot 为固定槽位编号，供动作掩码与特征定位。""" - - slot: int - kind: str # "clear" | "locate" | "probe" | "station" | "fallback" - position: Vec2 - channel: int | None = None - station_index: int | None = None - service_s: float = 0.0 # 预计服务耗时（含换频，不含移动） - targets: int = 1 # station: 待测频道数；fallback: 格数；其余 1 - - -@dataclass -class StepOutcome: - """一次高层决策-执行的客观结果（奖励折算由训练环境负责）。""" - - action: int - kind: str | None = None - channel: int | None = None - invalid: bool = False # 掩码外动作（无掩码消融路径）：未与模拟器交互 - delta_virtual_s: float = 0.0 - new_detections: tuple[int, ...] = () - info_gains: dict[int, float] = field(default_factory=dict) # 频道 -> 熵减(log) - clear_success: tuple[int, ...] = () - redundant: bool = False - position_reached: Vec2 | None = None - - -class HRLControllerMixin: - """高层调度混入：依赖宿主（HRLLoop）提供 BayesLoop 感知与执行原语。 - - 宿主须在 ``__init__`` 初始化：``_obs_mec_cache={}``、 - ``_station_pending_map={}``、``_initial_alive={channel: len(pf)}`` - （观测侧缓存与归一化基准；证书相关计算仍用 pf.mec() 精确值）。 - """ - - problem: int - # 由宿主 BayesLoop.__init__ 初始化；此处声明让 pylint 可见该诊断计数。 - no_shrink_measures: int = 0 - - # ------------------------------------------------------------ 候选构建 - def build_candidates(self) -> list[Candidate]: - """构建当前全部有效候选（固定槽位；训练/推理/heuristic 专家同源）。 - - 副作用：缓存每频道证书（``self._cert_cache``）与站点待测集 - （``self._station_pending_map``）供观测构建复用；调用方须在本 - 函数之后、任何状态变更之前构建观测。 - """ - config = self.config - detected = self.belief.channels_by_status(ChannelStatus.DETECTED) - by_slot: dict[int, Candidate] = {} - cert_cache: dict[int, object] = {} - station_map: dict[int, list[int]] = {} - - # --- clear 槽位：证书就绪且未因失败等待补测 - for channel in detected: - cert = self._best_certificate(channel) - cert_cache[channel] = cert - if cert is not None and cert.certified \ - and channel not in self._clear_failed: - by_slot[CLEAR_OFFSET + channel - 1] = Candidate( - CLEAR_OFFSET + channel - 1, "clear", tuple(cert.center), - channel=channel, - service_s=config.cost.clear_success_s, targets=1) - - # --- locate / probe 槽位：未就绪清除的已发现频道 - for channel in detected: - if CLEAR_OFFSET + channel - 1 in by_slot: - continue - region = self.belief.candidate_set(channel) - if region is None: - continue - blocked = self.blocked_probes.get(channel, []) \ - if self.problem == 4 else [] - if blocked: - point = self._hrl_probe_point(region, channel) - kind = "probe" - else: - point = self._select_point(channel) - kind = "locate" - if point is None: - continue - by_slot[LOCATE_OFFSET + channel - 1] = Candidate( - LOCATE_OFFSET + channel - 1, kind, tuple(point), channel=channel, - service_s=config.cost.measure_s + config.cost.switch_s, targets=1) - - # --- station 槽位：尚有 UNKNOWN 待测频道的覆盖站 - for index, point in enumerate(self.ledger.points): - pending = self._station_pending(index) - station_map[index] = pending - if not pending: - continue - switches = len(pending) - (1 if self.client.state.channel in pending else 0) - by_slot[STATION_OFFSET + index] = Candidate( - STATION_OFFSET + index, "station", tuple(point), - station_index=index, - service_s=(config.cost.measure_s * len(pending) - + config.cost.switch_s * max(switches, 0)), - targets=len(pending)) - - # --- fallback 槽位（仅问题4）：遮挡梯子耗尽且格数在预算内 - if self.problem == 4: - for channel in detected: - blocked = self.blocked_probes.get(channel, []) - if len(blocked) < config.search.max_blocked_probes: - continue - region = self.belief.candidate_set(channel) - if region is None: - continue - cells = fallback_clear_cells(region.polygon, config) - if not cells or len(cells) > config.search.fallback_max_cells: - continue - by_slot[FALLBACK_OFFSET + channel - 1] = Candidate( - FALLBACK_OFFSET + channel - 1, "fallback", - tuple(region.mec()[0]), channel=channel, - service_s=config.cost.clear_fail_s * len(cells), - targets=len(cells)) - - self._cert_cache = cert_cache - self._station_pending_map = station_map - self._last_candidates = list(by_slot.values()) - return self._last_candidates - - def candidate_by_slot(self, candidates: list[Candidate], - action: int) -> Candidate | None: - for cand in candidates: - if cand.slot == int(action): - return cand - return None - - # ------------------------------------------------------------ 动作掩码 - def hrl_action_mask(self, candidates: list[Candidate]) -> np.ndarray: - """槽位有效性掩码：无候选槽位恒屏蔽（训练/推理共用）。""" - mask = np.zeros(N_CANDIDATES, dtype=bool) - for cand in candidates: - mask[cand.slot] = True - return mask - - # ------------------------------------------------------------ 观测 - def _hrl_obs_mec(self, channel: int) -> tuple[Vec2, float]: - """观测用 MEC：幸存子采样上的最小包围圆，按 pf.version 缓存。 - - 仅作策略观测特征（近似 MEU/σ），证书相关计算一律用 pf.mec() - 精确值——两者分开缓存，互不影响。 - """ - pf = self.particles[channel] - cached = self._obs_mec_cache.get(channel) - if cached is not None and cached[0] == pf.version: - return cached[1], cached[2] - points = pf.subsample(600) - if not points: - return (0.0, 0.0), 0.0 - from ..geometry.localization import smallest_enclosing_circle - center, radius = smallest_enclosing_circle(points) - self._obs_mec_cache[channel] = (pf.version, center, radius) - return center, radius - - def _hrl_channel_status_index(self, channel: int) -> int: - """六状态编码：UNKNOWN/DETECTED/TRACKING/LOCATED/CLEARED/ABSENT。""" - status = self.belief.channels[channel].status - if status is ChannelStatus.UNKNOWN: - return 0 - if status is ChannelStatus.CLEARED: - return 4 - if status is ChannelStatus.ABSENT: - return 5 - cert = getattr(self, "_cert_cache", {}).get(channel) - if cert is not None and getattr(cert, "certified", False): - return 3 - _, radius = self._hrl_obs_mec(channel) - return 1 if radius > 500.0 else 2 - - def hrl_observation(self, candidates: list[Candidate]) -> np.ndarray: - """构造高层观测（shape=(HIGH_OBS_DIM,)，训练/推理逐位一致）。""" - config = self.config - field_cfg = config.field - position = self.client.state.position - belief = self.belief - - detected_count = len(belief.channels_by_status(ChannelStatus.DETECTED)) - accounted = sum(1 for ch in belief.channels.values() - if ch.status in (ChannelStatus.CLEARED, - ChannelStatus.ABSENT)) - cleared = len(self.cleared) - glob = [ - position[0] / POS_SCALE_M, - position[1] / POS_SCALE_M, - min(self.client.state.virtual_time_s, field_cfg.max_virtual_duration_s) - / field_cfg.max_virtual_duration_s, - cleared / 16.0, - accounted / CHANNEL_COUNT, - (self.client.state.channel - field_cfg.channel_min) - / max(field_cfg.channel_max - field_cfg.channel_min, 1), - float(self.problem == 4), - detected_count / 16.0, - ] - assert len(glob) == HIGH_GLOBAL_DIM - - pending_station = np.zeros(CHANNEL_COUNT + 1, dtype=bool) - for channels in getattr(self, "_station_pending_map", {}).values(): - for channel in channels: - pending_station[channel] = True - - channel_features: list[float] = [] - for channel in range(field_cfg.channel_min, field_cfg.channel_max + 1): - ch = belief.channels[channel] - status6 = [0.0] * 6 - status6[self._hrl_channel_status_index(channel)] = 1.0 - pf = self.particles[channel] - mec_visible = ch.status is ChannelStatus.DETECTED - if mec_visible: - center, radius = self._hrl_obs_mec(channel) - else: - center, radius = (0.0, 0.0), 0.0 - initial = max(self._initial_alive.get(channel, 1), 1) - channel_features.extend([ - *status6, - center[0] / POS_SCALE_M, - center[1] / POS_SCALE_M, - radius / RADIUS_SCALE_M, - math.log(len(pf) + 1) / math.log(initial + 1) - if initial > 1 else 0.0, - min(len(ch.measurements), MEASURE_COUNT_SCALE * 2) - / MEASURE_COUNT_SCALE / 2, - math.hypot(center[0] - position[0], center[1] - position[1]) - / POS_SCALE_M if mec_visible else 0.0, - len(self.blocked_probes.get(channel, [])) - / max(config.search.max_blocked_probes, 1) - if self.problem == 4 else 0.0, - 1.0 if (ch.status is ChannelStatus.UNKNOWN - and pending_station[channel]) else 0.0, - ]) - assert len(channel_features) == CHANNEL_COUNT * PER_CHANNEL_DIM - - by_slot = {cand.slot: cand for cand in candidates} - candidate_features: list[float] = [] - for slot in range(N_CANDIDATES): - cand = by_slot.get(slot) - if cand is None: - candidate_features.extend([0.0] * PER_CANDIDATE_DIM) - continue - dx, dy = cand.position[0] - position[0], cand.position[1] - position[1] - move_s = move_time(position, cand.position, config.cost) - candidate_features.extend([ - 1.0 if cand.kind == "clear" else 0.0, - 1.0 if cand.kind in ("locate", "probe") else 0.0, - 1.0 if cand.kind == "station" else 0.0, - 1.0 if cand.kind == "fallback" else 0.0, - dx / POS_SCALE_M, - dy / POS_SCALE_M, - min((move_s + cand.service_s) / COST_SCALE_S, 2.0), - min(cand.targets / 16.0, 1.0), - ]) - assert len(candidate_features) == N_CANDIDATES * PER_CANDIDATE_DIM - - observation = np.asarray(glob + channel_features + candidate_features, - dtype=np.float32) - assert observation.shape == (HIGH_OBS_DIM,) - return observation - - # ------------------------------------------------------------ 启发式专家 - def heuristic_action(self, candidates: list[Candidate]) -> int: - """把 BayesLoop 任务排序映射为槽位（BC 专家 + 无策略降级路径）。 - - 任务表与排序完全复用 :meth:`BayesLoop._build_tasks` / - :meth:`BayesLoop._order_tasks`；映射失败（如 Q4 梯子任务不在 - 任务表中）退化为最近有效候选。 - """ - by_slot = {cand.slot: cand for cand in candidates} - if not by_slot: - raise RuntimeError("heuristic_action：当前无可用候选") - try: - order = self._order_tasks(self._build_tasks()) - first = order[0] - if first.kind == "clear" and first.channel is not None: - slot = CLEAR_OFFSET + first.channel - 1 - elif first.kind == "locate" and first.channel is not None: - slot = LOCATE_OFFSET + first.channel - 1 - else: - slot = STATION_OFFSET + (first.station_index or 0) - if slot in by_slot: - return slot - except Exception: - pass - position = self.client.state.position - return min(by_slot, key=lambda s: ( - math.dist(position, by_slot[s].position), s)) - - # ------------------------------------------------------------ 决策执行 - def hrl_step(self, action: int, - candidates: list[Candidate] | None = None) -> StepOutcome: - """执行一次高层决策；candidates 为空则现场构建（训练环境逐位同源）。""" - action = int(action) - self._reset_step_recording() - old_virtual = self.client.state.virtual_time_s - if candidates is None: - candidates = self.build_candidates() - cand = self.candidate_by_slot(candidates, action) - if cand is None: - return StepOutcome(action=action, invalid=True) - if self.stop_reason is not None: - return StepOutcome(action=action, kind=cand.kind, - channel=cand.channel, invalid=True) - - if cand.kind == "station": - self._execute_station(cand.station_index) - elif cand.kind in ("locate", "probe"): - self._hrl_execute_locate(cand) - elif cand.kind == "clear": - self._hrl_execute_clear(cand) - elif cand.kind == "fallback": - self._hrl_execute_fallback(cand) - else: # pragma: no cover - 布局守卫 - raise ValueError(f"未知候选类型: {cand.kind}") - - outcome = StepOutcome(action=action, kind=cand.kind, - channel=cand.channel, - position_reached=tuple(self.client.state.position)) - outcome.delta_virtual_s = self.client.state.virtual_time_s - old_virtual - new_detections: list[int] = [] - info_gains: dict[int, float] = {} - redundant = False - for event in self._step_events: - if event["was_unknown"] and event["result"] in (DIRECTION, NEAR): - new_detections.append(event["channel"]) - gain = math.log10(event["pf_before"] + 1) - math.log10(event["pf_after"] + 1) - info_gains[event["channel"]] = (info_gains.get(event["channel"], 0.0) - + min(max(gain, 0.0), 2.0)) - redundant = redundant or event["redundant"] - outcome.new_detections = tuple(new_detections) - outcome.info_gains = info_gains - outcome.redundant = redundant - outcome.clear_success = tuple(self._step_clears) - self._step_clears = [] - return outcome - - # ------------------------------------------------------------ 执行原语 - def _hrl_execute_locate(self, cand: Candidate) -> None: - """定位测量（含 Q4 梯子探点）：记录证据，不自动清除（高层决策）。""" - config = self.config - channel = cand.channel - pf = self.particles[channel] - before = len(pf) - outcome = self.client.measure(cand.position, channel) - self._record_measure(channel, cand.position, outcome) - self._may_clear.add(channel) - if self.problem == 4: - if outcome.result == NO_SIGNAL: - if channel not in self.directional_observed: - region = self.belief.candidate_set(channel) - safe = safe_disk_radius(region, config) if region else -1.0 - if safe >= 0 and math.dist(cand.position, region.mec()[0]) <= safe: - self.directional_observed.append(channel) - self.blocked_probes.setdefault(channel, []).append(cand.position) - else: - self.blocked_probes.pop(channel, None) - if len(pf) == before: - self._stall[channel] = self._stall.get(channel, 0) + 1 - self.no_shrink_measures += 1 - else: - self._stall[channel] = 0 - - def _hrl_execute_clear(self, cand: Candidate) -> None: - """证书清除：证书失效（不应被掩码允许）时静默跳过。""" - cert = self._best_certificate(cand.channel) - if cert is None or not cert.certified: - self._anomaly("hrl_clear_without_certificate", cand.channel) - return - if self._attempt_clear(cand.channel, cert.center, cert): - self._clear_failed.discard(cand.channel) - self._stall.pop(cand.channel, None) - else: - self._clear_failed.add(cand.channel) - self._may_clear.discard(cand.channel) - - def _hrl_execute_fallback(self, cand: Candidate) -> None: - """Q4 后备网格清除（与 DirectionalExtension._fallback_clear 同口径）。""" - channel = cand.channel - region = self.belief.candidate_set(channel) - if region is None: - return - config, session = self.config, self.config.session - cells = fallback_clear_cells(region.polygon, config) - entry = {"channel": channel, "cells": len(cells)} - self.fallback_log.append(entry) - if len(cells) > config.search.fallback_max_cells: - entry["outcome"] = "skipped_cell_cap" - return - remaining = self.client.real_remaining_s - if remaining is not None and len(cells) * session.real_rtt_p95_s \ - + session.reserve_exit_s > remaining: - entry["outcome"] = "skipped_budget" - return - remaining_cells = list(cells) - attempts = 0 - while remaining_cells: - if self._virtual_cap_reached(): - entry["outcome"] = "skipped_virtual_budget" - entry["attempts"] = attempts - return - cell = min(remaining_cells, key=lambda c: ( - math.dist(self.client.state.position, c), c)) - remaining_cells.remove(cell) - attempts += 1 - outcome = self.client.clear(cell, channel) - if outcome.result == CLEAR_SUCCESS: - self.belief.mark_cleared(channel) - self.cleared.append(channel) - self._step_clears.append(channel) - self.blocked_probes.pop(channel, None) - entry["outcome"] = "cleared" - entry["attempts"] = attempts - return - entry["outcome"] = "exhausted" - entry["attempts"] = attempts - self._anomaly("hrl_fallback_exhausted", channel, cells=len(cells)) - - def _hrl_probe_point(self, region, channel: int) -> Vec2: - """定向遮挡梯子探点（与 DirectionalExtension._next_probe 同口径）。""" - from ..planner import select_measurement_point - blocked = self.blocked_probes.get(channel, []) - center, radius = region.mec() - if not blocked: - return select_measurement_point(region, self.client.state.position, - self.config).position - safe = safe_disk_radius(region, self.config) - first = blocked[0] - if len(blocked) == 1 and safe >= 0: - dx, dy = center[0] - first[0], center[1] - first[1] - norm = math.hypot(dx, dy) - if norm <= 1e-9: - return (center[0] + safe, center[1]) - scale = min(1.0, safe / norm) - return (center[0] + dx * scale, center[1] + dy * scale) - base = math.atan2(first[1] - center[1], first[0] - center[0]) - azimuth = base + math.radians(45.0 * (len(blocked) - 1)) - return (center[0] + radius * math.cos(azimuth), - center[1] + radius * math.sin(azimuth)) - - # ------------------------------------------------------------ 事件捕获 - def _reset_step_recording(self) -> None: - self._step_events = [] - self._step_clears = [] - - def _record_event(self, **event) -> None: - self._step_events.append(event) diff --git a/src/mcm26b/planner/hrl_loop.py b/src/mcm26b/planner/hrl_loop.py deleted file mode 100644 index 6918301..0000000 --- a/src/mcm26b/planner/hrl_loop.py +++ /dev/null @@ -1,172 +0,0 @@ -"""P7 分层强化学习闭环（``--strategy hrl`` / ``--q4-strategy hrl``）。 - -边界与保守性声明（与 rl_loop.py 同口径）： -- 高层只在固定候选槽位上做调度决策（证书清除 / 信息增益定位 / 覆盖站 / - Q4 梯子探点与后备清除），候选几何全部由既有规则生成； -- 感知（belief + 网格幸存集）、清除证书、跳测规则、异常留痕完全继承 - :class:`BayesLoop`，不触碰真源保留定理的前提； -- 问题3 的幸存集按全部读数更新（no_signal 删盘）；问题4 定向源的 - no_signal 不能排除朝向遮挡，幸存集只按 direction/near 更新，遮挡 - 证据走 blocked_probes 梯子与后备清除槽位（DirectionalExtension 口径）； -- 高层策略缺失时经 :meth:`heuristic_action` 降级为 BayesLoop 任务排序， - 闭环仍然完备可运行。 - -问题4 发现覆盖使用 49 点网格（七点环的覆盖证明只对全向源成立）。 -""" - -from __future__ import annotations - -from ..belief import ChannelStatus -from ..config import ProblemConfig, Vec2 -from ..coverage import CoverageLedger, coverage_points -from ..hrl.scheduler import HRLControllerMixin, point_key -from ..protocol.rules import NO_SIGNAL -from .bayes_loop import BayesLoop - - -class HRLLoop(HRLControllerMixin, BayesLoop): - """分层闭环：BayesLoop 感知/证书 + 高层槽位调度（问题3/4 共用）。""" - - def __init__(self, client, config: ProblemConfig = ProblemConfig(), *, - problem: int = 3, high_policy=None, low_policy=None, - trace_executor: bool = False): - super().__init__(client, config) - if problem not in (3, 4): - raise ValueError("problem 必须是 3 或 4") - self.problem = int(problem) - self.high_policy = high_policy - self.low_policy = low_policy - self.trace_executor = bool(trace_executor) - self.executor_trace: list[dict] = [] - self.hrl_decisions: list[dict] = [] - self._step_events: list[dict] = [] - self._step_clears: list[int] = [] - self._cert_cache: dict = {} - self._last_candidates: list = [] - self._obs_mec_cache: dict = {} - self._station_pending_map: dict = {} - # 观测里幸存格数的归一化基准（初始全集），使特征对训练/部署的 - # 网格密度差不变。 - self._initial_alive = {channel: len(pf) - for channel, pf in self.particles.items()} - if problem == 4: - self.directional_observed: list[int] = [] - self.blocked_probes: dict[int, list[Vec2]] = {} - self.fallback_log: list[dict] = [] - # 49 点网格：七点环覆盖证明只对全向源成立 [口径见模块注释] - self.ledger = CoverageLedger(config, points=coverage_points(config)) - - # ------------------------------------------------------------ 主循环 - def _coverage_phase(self) -> None: - """统一任务循环：每步构建候选，由高层策略（或启发式降级）选槽执行。""" - while self.stop_reason is None: - if self._count_limit_reached() or self._virtual_cap_reached(): - return - # 顺路观测会改变信念、证书和候选；必须发生在本轮快照之前。 - if self.high_policy is None and self.problem == 3 \ - and self.config.bayes.opportunistic_measure: - self._measure_at_stop() - if self.stop_reason is not None or self._count_limit_reached() \ - or self._virtual_cap_reached(): - return - candidates = self.build_candidates() - if not candidates: - return - if self.high_policy is not None: - observation = self.hrl_observation(candidates) - mask = self.hrl_action_mask(candidates) - action = self.high_policy.select(observation, mask) - else: - # 降级路径：BayesLoop 任务排序（无 RL 依赖时闭环仍完备） - action = self.heuristic_action(candidates) - if self.candidate_by_slot(candidates, action) is None: - # 非法策略输出不能造成零耗时无限循环；事实记入异常再安全调度。 - self._anomaly("hrl_invalid_policy_action", 0, action=int(action)) - action = self.heuristic_action(candidates) - cand = self.candidate_by_slot(candidates, action) - if cand is not None: - self._log_low_level(cand) - self.hrl_decisions.append({ - "action": int(action), "kind": cand.kind, - "channel": cand.channel, - "virtual_time_s": self.client.state.virtual_time_s}) - self.hrl_step(action, candidates=candidates) - - # ------------------------------------------------------------ 感知记录 - def _record_measure(self, channel: int, position: Vec2, outcome) -> None: - """BayesLoop._record_measure 的事件捕获版；问题4 no_signal 不删格。 - - Q4 语义：no_signal 可能是朝向遮挡 [观测推理]，删除幸存格会破坏 - 真源保留，因此问题4 幸存集只按 direction/near 更新；缺席证明 - 交由 49 点网格覆盖账本（继承的 _mark_absent_after_coverage）。 - """ - key = point_key(position) - redundant = key in self._measured_points.get(channel, set()) - belief = self.belief.channels[channel] - was_unknown = belief.status is ChannelStatus.UNKNOWN - pf = self.particles[channel] - before = len(pf) - self.belief.update_measure(channel, position, outcome.result, - outcome.svd_deg) - self._measured_points.setdefault(channel, set()).add(key) - self._clear_failed.discard(channel) - belief = self.belief.channels[channel] - if belief.status not in (ChannelStatus.CLEARED, ChannelStatus.ABSENT): - if not (self.problem == 4 and outcome.result == NO_SIGNAL): - pf.apply_measure(position, outcome.result, outcome.svd_deg) - if not pf: - if belief.status is ChannelStatus.UNKNOWN: - self.belief.mark_absent(channel) - if channel not in self.pf_absent_channels: - self.pf_absent_channels.append(channel) - else: - self._anomaly("pf_empty_while_detected", channel, - position=position) - self._record_event(channel=channel, result=outcome.result, - pf_before=before, pf_after=len(pf), - redundant=redundant, was_unknown=was_unknown) - - def _attempt_clear(self, channel: int, target: Vec2, cert) -> bool: - success = super()._attempt_clear(channel, target, cert) - if success and hasattr(self, "blocked_probes"): - self.blocked_probes.pop(channel, None) - if success: - self._step_clears.append(channel) - return success - - # ------------------------------------------------------------ 低层执行留痕 - def _log_low_level(self, cand) -> None: - """低层到达推演留痕（本地步进，不与模拟器交互，不产生虚拟耗时）。""" - if not self.trace_executor: - return - from ..hrl.executor import LowLevelExecutor - executor = getattr(self, "_executor", None) - if executor is None: - executor = self._executor = LowLevelExecutor(self.low_policy) - waypoints = executor.roll(self.client.state.position, cand.position) - if self.low_policy is not None: - self.executor_trace.append({ - "kind": cand.kind, "channel": cand.channel, - "goal": [round(v, 2) for v in cand.position], - "waypoints": [[round(v, 2) for v in p] for p in waypoints], - "navigation": dict(executor.last_roll_info), - }) - - # ------------------------------------------------------------ 收尾 - def _summary(self, info) -> dict: - report = super()._summary(info) - report["strategy"] = "hrl_q4" if self.problem == 4 else "hrl" - report["hrl_problem"] = self.problem - kinds: dict[str, int] = {} - for decision in self.hrl_decisions: - kinds[decision["kind"]] = kinds.get(decision["kind"], 0) + 1 - report["hrl_action_kinds"] = kinds - report["hrl_decisions"] = self.hrl_decisions - report["hrl_executor_trace"] = self.executor_trace - if self.problem == 4: - report["directional_observed_channels"] = \ - list(self.directional_observed) - report["fallback_clears"] = self.fallback_log - report["fallback_clear_attempts"] = sum( - entry.get("attempts", 0) for entry in self.fallback_log) - return report diff --git a/src/mcm26b/planner/rl_loop.py b/src/mcm26b/planner/rl_loop.py deleted file mode 100644 index c687204..0000000 --- a/src/mcm26b/planner/rl_loop.py +++ /dev/null @@ -1,94 +0,0 @@ -"""P6b 强化学习闭环接入：PPO 学习"下一检测点"选择（问题3/4）。 - -边界与保守性声明： -- RL 只替换测量点排序（在既有几何候选集内做 argmax），不生成新的 - 空间候选、不修改覆盖证明、不修改清除证书； -- 问题3：继承 :class:`BayesLoop`（七点覆盖 + 粒子滤波 + 滚动任务调度）， - 仅 :meth:`_select_point` 改由 PPO 输出候选编号； -- 问题4：继承 :class:`DirectionalExtension`（49 点定向覆盖 + 对跖/环探 + - 20m 有限圆盘覆盖后备清除），仅 :meth:`_next_probe` 改由 PPO 输出； - 遮挡计数、定向观测与后备清除兜底完全保留； -- 训练环境与运行特征同源（mcm26b.rl.features），模型加载时校验 - 观测/动作维度，不匹配即拒绝运行，避免静默错位。 - -默认模型位置：output/rl/q3_ppo.zip、output/rl/q4_ppo.zip。 -""" - -from __future__ import annotations - -from pathlib import Path - -from ..config import ProblemConfig, Vec2 -from ..rl.policy import PointSelectionPolicy, default_model_path -from . import DirectionalExtension -from .bayes_loop import BayesLoop - - -class RLBayesLoop(BayesLoop): - """问题3：贝叶斯闭环骨架 + PPO 定位选点（--strategy rl）。""" - - def __init__(self, client, config: ProblemConfig = ProblemConfig(), *, - model_path: str | Path | None = None, device: str = "auto"): - super().__init__(client, config) - self.rl_model_path = Path(model_path) if model_path else default_model_path(3) - self.rl_policy = PointSelectionPolicy(self.rl_model_path, device=device) - self.rl_probe_actions: list[dict] = [] - - def _select_point(self, channel: int) -> Vec2 | None: - region = self.belief.candidate_set(channel) - if region is None: - return None - point, action = self.rl_policy.select( - region, self.client.state.position, channel, self.config, - problem=3, blocked=[], belief=self.belief.channels[channel], - virtual_time_s=self.client.state.virtual_time_s, - receiver_channel=self.client.state.channel) - self.rl_probe_actions.append({ - "channel": channel, "action": int(action), - "position": [round(point[0], 3), round(point[1], 3)], - "virtual_time_s": self.client.state.virtual_time_s, - }) - return point - - def _summary(self, info) -> dict: - report = super()._summary(info) - report["strategy"] = "rl" - report["rl_model"] = str(self.rl_model_path) - report["rl_device"] = self.rl_policy.device - report["rl_probe_actions"] = self.rl_probe_actions - return report - - -class RLDirectionalLoop(DirectionalExtension): - """问题4：定向闭环骨架 + PPO 遮挡梯子选点（--q4-strategy rl）。""" - - def __init__(self, client, config: ProblemConfig = ProblemConfig(), *, - model_path: str | Path | None = None, device: str = "auto"): - super().__init__(client, config) - self.rl_model_path = Path(model_path) if model_path else default_model_path(4) - self.rl_policy = PointSelectionPolicy(self.rl_model_path, device=device) - self.rl_probe_actions: list[dict] = [] - - def _next_probe(self, region, channel: int) -> Vec2: - blocked = self.blocked_probes.get(channel, []) - point, action = self.rl_policy.select( - region, self.client.state.position, channel, self.config, - problem=4, blocked=blocked, - belief=self.belief.channels[channel], - virtual_time_s=self.client.state.virtual_time_s, - receiver_channel=self.client.state.channel) - self.rl_probe_actions.append({ - "channel": channel, "action": int(action), - "position": [round(point[0], 3), round(point[1], 3)], - "blocked_count": len(blocked), - "virtual_time_s": self.client.state.virtual_time_s, - }) - return point - - def _summary(self, info) -> dict: - report = super()._summary(info) - report["strategy"] = "rl_q4" - report["rl_model"] = str(self.rl_model_path) - report["rl_device"] = self.rl_policy.device - report["rl_probe_actions"] = self.rl_probe_actions - return report diff --git a/src/mcm26b/process_control.py b/src/mcm26b/process_control.py deleted file mode 100644 index 2de7e51..0000000 --- a/src/mcm26b/process_control.py +++ /dev/null @@ -1,175 +0,0 @@ -"""跨平台子进程控制：启动隔离、协作式暂停/停止、必要时强杀整棵进程树。 - -训练前端要在 POSIX 与 Windows 上表现一致，而两边的进程模型差别正好卡在 -「停止训练」这条路径上： - -- ``os.killpg`` / ``os.getpgid`` **只存在于 POSIX**；Windows 上访问它们会抛 - ``AttributeError``。旧实现只捕获 ``ProcessLookupError`` / ``PermissionError``， - 于是 ``POST /api/train/stop`` 在 Windows 上直接 500，训练进程照跑——表现就是 - 「点停止没反应」。 -- ``subprocess.Popen(start_new_session=True)`` 在 Windows 上被**静默忽略**： - CPython 的 Windows ``_execute_child`` 形参名就是 ``unused_start_new_session``， - 要隔离进程组只能传 ``CREATE_NEW_PROCESS_GROUP``。 -- POSIX 的 SIGTERM 在 Windows 上经 ``os.kill`` 会退化成 ``TerminateProcess``： - 硬杀，训练脚本里"收到信号后保存模型"的处理器根本不会执行。 - -所以「优雅暂停 / 优雅停止」不依赖信号量语义，而是走一个**控制文件**：前端原子 -写 JSON，训练脚本每步轮询。两端都只是普通文件 IO，Windows 与 POSIX 行为一致； -信号与强杀退居兜底，只在训练进程不响应控制文件时使用。 -""" - -from __future__ import annotations - -import json -import os -import signal -import subprocess -import time -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -__all__ = [ - "IS_WINDOWS", "ControlState", "spawn_kwargs", "write_control", - "read_control", "write_json_atomic", "terminate_process_tree", -] - -# 运行期常量而非"导入期分支"：调用方（含测试）可以 monkeypatch 它来模拟 Windows -IS_WINDOWS = os.name == "nt" - -# 只有 Windows 定义 CREATE_NEW_PROCESS_GROUP；POSIX 上取 0 表示不使用 creationflags -_CREATE_NEW_PROCESS_GROUP = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) - -# 发出「停止」到强杀之间留给训练脚本收尾（保存模型 + 写元数据）的秒数 -DEFAULT_GRACE_S = 15.0 - - -def spawn_kwargs() -> dict[str, Any]: - """返回 ``Popen`` 的平台相关参数：让训练进程独立于前端进程组。 - - POSIX 用 ``setsid``（``start_new_session``）；Windows 上用 - ``CREATE_NEW_PROCESS_GROUP``，否则前端所在控制台的 Ctrl+C 会连带打到训练进程。 - """ - if IS_WINDOWS: - if _CREATE_NEW_PROCESS_GROUP: - return {"creationflags": _CREATE_NEW_PROCESS_GROUP} - return {} - return {"start_new_session": True} - - -@dataclass(frozen=True) -class ControlState: - """控制文件的一次快照；缺失或损坏时字段全为默认（即"无指令"）。""" - - paused: bool = False - stop: bool = False - seq: int = 0 - ts: float = 0.0 - - -def write_control(path: Path, *, paused: bool = False, stop: bool = False, - seq: int | None = None) -> ControlState: - """原子写控制文件（同目录临时文件 + ``os.replace``）。 - - 训练侧随时可能来读，必须避免读到写了一半的 JSON。``os.replace`` 在 - Windows 与 POSIX 上都是原子的（同目录、同卷）。 - """ - path = Path(path) - path.parent.mkdir(parents=True, exist_ok=True) - state = ControlState( - paused=bool(paused), - stop=bool(stop), - seq=int(seq) if seq is not None else time.time_ns(), - ts=time.time(), - ) - payload = {"paused": state.paused, "stop": state.stop, - "seq": state.seq, "ts": state.ts} - tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp") - tmp.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") - os.replace(tmp, path) - return state - - -def read_control(path: Path) -> ControlState: - """读控制文件；缺失 / 半截 / 损坏一律当作"无指令"，绝不让训练崩在控制面上。""" - try: - raw = Path(path).read_text(encoding="utf-8") - payload = json.loads(raw) - except (OSError, ValueError, UnicodeDecodeError): - return ControlState() - if not isinstance(payload, dict): - return ControlState() - return ControlState( - paused=bool(payload.get("paused", False)), - stop=bool(payload.get("stop", False)), - seq=int(payload.get("seq") or 0), - ts=float(payload.get("ts") or 0.0), - ) - - -def write_json_atomic(path: Path, payload: Any) -> None: - """原子写 JSON 文件（同目录临时文件 + ``os.replace``）。 - - 用于"急停恢复"相关的标记文件（``resume.json`` / ``forced_stop.json`` 等）： - 读取方随时可能来读，必须避免看到写了一半的内容。 - """ - path = Path(path) - path.parent.mkdir(parents=True, exist_ok=True) - tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp") - tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2), - encoding="utf-8") - os.replace(tmp, path) - - -def terminate_process_tree(process: subprocess.Popen | None, *, - grace_s: float = DEFAULT_GRACE_S, - poll_s: float = 0.2) -> bool: - """等待 ``grace_s`` 让协作式停止生效，超时则强杀整棵进程树。 - - 返回**是否动用了强杀**：``False`` 表示进程在宽限期内自行退出（模型有机会 - 落盘），``True`` 表示进程被强杀、最后一段产物可能缺失。调用方应先通过控制 - 文件请求停止，再调用本函数；本函数不负责写控制文件。 - """ - if process is None: - return False - if process.poll() is not None: - return False - deadline = time.monotonic() + max(0.0, grace_s) - while time.monotonic() < deadline: - if process.poll() is not None: - return False - time.sleep(min(poll_s, max(0.0, deadline - time.monotonic()))) - if process.poll() is not None: - return False - _hard_kill_tree(process) - try: - process.wait(timeout=10.0) - except subprocess.TimeoutExpired: - pass - return True - - -def _hard_kill_tree(process: subprocess.Popen) -> None: - """强杀进程树：POSIX 走进程组 SIGKILL，Windows 走 ``taskkill /T``。 - - 这里所有 POSIX 专属调用都走 ``getattr`` 探测，Windows 上拿不到分支就退回 - ``Popen.kill()``——不会再出现 ``module 'os' has no attribute 'killpg'``。 - """ - if IS_WINDOWS: - try: - subprocess.run(["taskkill", "/F", "/T", "/PID", str(process.pid)], - capture_output=True, check=False) - return - except OSError: - process.kill() - return - killpg = getattr(os, "killpg", None) - getpgid = getattr(os, "getpgid", None) - sigkill = getattr(signal, "SIGKILL", None) - if killpg is not None and getpgid is not None and sigkill is not None: - try: - killpg(getpgid(process.pid), sigkill) - return - except (ProcessLookupError, PermissionError, OSError): - pass - process.kill() diff --git a/src/mcm26b/rl/__init__.py b/src/mcm26b/rl/__init__.py deleted file mode 100644 index b5a1480..0000000 --- a/src/mcm26b/rl/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -"""问题3/4 强化学习扩展（可选依赖组 ``rl``）。 - -- features.py：观测/候选动作的纯几何接口（numpy），训练与运行共享； -- env.py：单源定位 Gymnasium 训练环境（依赖 referee，仅训练侧加载）； -- policy.py：PPO 模型加载与确定性推理（惰性加载 torch/sb3）。 - -运行闭环默认不导入 torch/stable-baselines3；只有 CLI 显式选择 -``--strategy rl``（问题3）或 ``--q4-strategy rl``（问题4）时， -``planner.rl_loop`` 才构造 :class:`policy.PointSelectionPolicy`。 -""" - -from .features import candidate_actions, build_observation - -__all__ = ["candidate_actions", "build_observation"] diff --git a/src/mcm26b/rl/_constants.py b/src/mcm26b/rl/_constants.py deleted file mode 100644 index 970a425..0000000 --- a/src/mcm26b/rl/_constants.py +++ /dev/null @@ -1,11 +0,0 @@ -"""RL 观测/动作空间常量：模型接口的唯一字面量来源。 - -布局说明见 features.py；这些数字由训练脚本写入模型接口， -改动其中任一值都会使已训练模型失配，须同步重训。 -""" - -GLOBAL_DIM = 41 -PER_ACTION_DIM = 6 -NUM_ACTIONS = 60 -LADDER_SLOTS = 8 -OBS_DIM = GLOBAL_DIM + NUM_ACTIONS * PER_ACTION_DIM # 401 diff --git a/src/mcm26b/rl/env.py b/src/mcm26b/rl/env.py deleted file mode 100644 index dfd5c27..0000000 --- a/src/mcm26b/rl/env.py +++ /dev/null @@ -1,331 +0,0 @@ -"""问题3/4 的强化学习训练环境：单源"测向—定位—清除"子任务。 - -环境与自建评测机 referee 共享同一边界（内存回环，与真实 HTTP 同协议）， -观测只来自 :class:`mcm26b.belief.BeliefState` 与协议响应；真值仅在 reset -时用于布设课程场景，不进入观测。策略端看到的接口与闭环运行时一致。 - -问题抽象：机器狗已获得某频道首次 direction 读数，需要在一组几何可行候选 -检测点中选择下一个测点，目标是尽快把候选集最小包围圆压到 20m 清除证书 -阈值并清除成功。问题4额外引入定向源遮挡：保证接收盘内出现 no_signal 是 -"朝向遮挡"的观测证据，环境记录 blocked 序列并开放环探候选。 - -该环境是训练侧组件（依赖 referee + gymnasium），运行时代码不导入本模块； -导入依赖由可选组 ``[dependency-groups] rl`` 提供。 -""" - -from __future__ import annotations - -import math -from random import Random - -import gymnasium as gym -import numpy as np -from gymnasium import spaces - -from ..belief import BeliefState -from ..config import ProblemConfig -from ..planner import clear_certificate, fallback_clear_cells, safe_disk_radius -from ..protocol.client import SimulatorClient -from ..protocol.log import SessionLog -from ..protocol.rules import CLEAR_SUCCESS, DIRECTION, NO_SIGNAL -from .features import NUM_ACTIONS, OBS_DIM, action_mask, build_observation, candidate_actions - - -class LocalizationEnv(gym.Env): - """单源定位清除 Gymnasium 环境。 - - problem=3：全向源；problem=4：定向/全向混合（定向概率 0.5，训练分布 - 比题面更稠密以覆盖遮挡分支；正式策略仍由闭环的后备清除兜底）。 - """ - - metadata = {"render_modes": []} - - def __init__(self, config: ProblemConfig | None = None, *, - problem: int = 3, max_episode_steps: int = 16, - directional_probability: float = 0.5, - real_budget_s: float = 3600.0, - error_seed: int | None = None, - handoff_probability: float = 0.35, boundary_probability: float = 0.2): - super().__init__() - if problem not in (3, 4): - raise ValueError("problem 必须是 3 或 4") - self.config = config or ProblemConfig() - self.problem = int(problem) - self.max_episode_steps = int(max_episode_steps) - self.directional_probability = float(directional_probability) - self.real_budget_s = float(real_budget_s) - self.error_seed = error_seed - self.handoff_probability = handoff_probability - self.boundary_probability = boundary_probability - self.observation_space = spaces.Box(low=-10.0, high=10.0, - shape=(OBS_DIM,), dtype=np.float32) - self.action_space = spaces.Discrete(NUM_ACTIONS) - - # 惰性导入：本模块只属于训练/评估侧，运行闭环不加载 referee。 - from referee import LocalSimulator as _RefereeLocalSimulator - from referee import Source - from ..harness import LocalSimTransport, referee_spec - self._source_cls = Source - self._local_simulator_cls = _RefereeLocalSimulator - self._local_sim_transport_cls = LocalSimTransport - self._referee_spec = referee_spec - - self._scene_rng = Random() - self._simulator = None - self._client = None - self._source = None - self._belief: BeliefState | None = None - self._channel: int = 1 - self._blocked: list = [] - self._directional_observed = False - self._clear_failed = False - self._steps = 0 - self._terminated = False - self._truncated = False - self._measured_points: set = set() - self._fallback_attempts = 0 - - # ------------------------------------------------------------------ 场景 - def _make_source_and_probe(self): - """用真值布设一个"首次测向必为 direction"的课程场景（真值不进观测）。""" - rng = self._scene_rng - field = self.config.field - channel = rng.randint(field.channel_min, field.channel_max) - radius = field.omega_radius_m * math.sqrt(rng.random()) - edge = rng.random() < self.boundary_probability - if edge: - radius = field.omega_radius_m * rng.uniform(0.98, 1.0) - angle = rng.uniform(0.0, 2 * math.pi) - x, y = radius * math.cos(angle), radius * math.sin(angle) - r_eff = rng.uniform(field.r_eff_min_m, field.r_eff_max_m) - if edge: - r_eff = field.r_eff_min_m - directional = self.problem == 4 and rng.random() < self.directional_probability - heading = rng.uniform(0.0, 360.0) if directional else 0.0 - - # 首次检测点：距离 [300, min(1450, r_eff-20)] 内随机，定向源取朝向 - # ±89.9°（接近但不跨越90°边界，保证direction）。机器狗可到Ω外 [题面]。 - max_distance = max(310.0, min(1450.0, r_eff - 20.0)) - max_distance = max(max_distance, 310.0) - probe_distance = rng.uniform(300.0, max_distance) - probe_bearing = (heading + rng.uniform(-89.9, 89.9) - if directional else rng.uniform(0.0, 360.0)) - probe_radians = math.radians(probe_bearing) - probe = (x + probe_distance * math.cos(probe_radians), - y + probe_distance * math.sin(probe_radians)) - source = self._source_cls(channel=channel, x=x, y=y, r_eff_m=r_eff, - directional=directional, heading_deg=heading) - return source, probe - - def _make_episode(self) -> bool: - """重建模拟器并完成首次测向；返回 False 表示连续未得到 direction。""" - for _ in range(20): - self._source, probe = self._make_source_and_probe() - self._simulator = self._local_simulator_cls( - [self._source], robot_id="rl-train", - spec=self._referee_spec( - self.config, max_real_duration_s=self.real_budget_s), - error_seed=(self.error_seed if self.error_seed is not None - else self._scene_rng.randint(0, 2 ** 31 - 1)), - stress_mode=True) - self._client = SimulatorClient( - "rl-train", config=self.config, - transport=self._local_sim_transport_cls(self._simulator), - log=SessionLog()) - self._client.enter() - outcome = self._client.measure(probe, self._source.channel) - if outcome.result == DIRECTION: - self._channel = self._source.channel - self._belief = BeliefState(self.config) - self._belief.update_measure(self._channel, probe, outcome.result, - outcome.svd_deg) - self._blocked = [] - self._directional_observed = False - self._clear_failed = False - self._measured_points = {self._point_key(probe)} - self._fallback_attempts = 0 - # 运行中可能先服务其他频道后再定位本源；模拟这种位置/频道交接。 - if self._scene_rng.random() < self.handoff_probability: - angle = self._scene_rng.uniform(0, 2*math.pi) - reach = self.config.field.omega_radius_m * math.sqrt(self._scene_rng.random()) - other = (self.config.field.channel_min - if self._channel != self.config.field.channel_min - else self.config.field.channel_max) - self._client.measure((reach*math.cos(angle), reach*math.sin(angle)), other) - self._start_virtual_s = self._client.state.virtual_time_s - return True - # near / no_signal 首测属于概率为零的边界或课程构造失败，换场景。 - continue - return False - - # ------------------------------------------------------------------ Gym API - @staticmethod - def _point_key(point) -> tuple[int, int]: - return (int(round(point[0] * 2)), int(round(point[1] * 2))) - - def reset(self, *, seed: int | None = None, options: dict | None = None): - super().reset(seed=seed) - self.close() - if seed is not None: - self._scene_rng = Random(seed) - self._steps = 0 - self._localize_steps = 0 - self._failure_reason = None - self._terminated = False - self._truncated = False - if not self._make_episode(): - raise RuntimeError("连续 20 次无法构造 direction 首测场景") - observation = self._observe() - return observation, self._info() - - def _observe(self) -> np.ndarray: - region = self._belief.candidate_set(self._channel) - if region is None: - raise RuntimeError("首次测向后信念状态缺失候选集") - return build_observation( - region, self._client.state.position, self._channel, self.config, - self.problem, self._blocked, self._belief.channels[self._channel], - virtual_time_s=self._client.state.virtual_time_s, - receiver_channel=self._client.state.channel) - - def action_masks(self): - return action_mask(self._belief.candidate_set(self._channel), self._client.state.position, - self.config, self.problem, self._blocked, - self._belief.channels[self._channel]) - - def _info(self) -> dict: - source = self._source - return { - "channel": self._channel, - "source": {"x": source.x, "y": source.y, "r_eff_m": source.r_eff_m, - "directional": source.directional, - "heading_deg": source.heading_deg}, - "steps": self._steps, - "blocked_count": len(self._blocked), - "directional_observed": self._directional_observed, - "fallback_attempts": self._fallback_attempts, - "virtual_cost_s": self._client.state.virtual_time_s - self._start_virtual_s, - "failure_reason": self._failure_reason, - } - - def step(self, action): - if self._terminated or self._truncated: - raise RuntimeError("step() 需要在 terminated/truncated 后先 reset()") - if not 0 <= int(action) < NUM_ACTIONS: - raise ValueError(f"动作 {action} 超出 Discrete({NUM_ACTIONS})") - action = int(action) - self._steps += 1 - region = self._belief.candidate_set(self._channel) - if region is None: - return self._finish(observation=self._observe(), reward=-20.0, - terminated=True, truncated=False) - - # 达到清除证书且上次清除失败已被新测量重置：先自动清除（闭环语义）。 - cert = clear_certificate(region, self._channel, self.config) - if cert.certified and not self._clear_failed: - return self._try_clear(cert, observation=self._observe()) - - old_virtual = self._client.state.virtual_time_s - candidates = candidate_actions(region, self._client.state.position, - self.config, self._blocked) - point = candidates[action] - try: - outcome = self._client.measure(point, self._channel) - self._belief.update_measure(self._channel, point, outcome.result, - outcome.svd_deg) - except Exception as exc: - self._failure_reason = type(exc).__name__ + ': ' + str(exc) - return self._finish(observation=self._observe(), reward=-100.0, - terminated=True, truncated=False) - - cost_s = self._client.state.virtual_time_s - old_virtual - reward = -cost_s / 100.0 - self._measured_points.add(self._point_key(point)) - new_region = self._belief.candidate_set(self._channel) - if outcome.result == NO_SIGNAL and self.problem == 4: - center = region.mec()[0] - safe = safe_disk_radius(region, self.config) - if safe >= 0 and math.dist(point, center) <= safe + 1e-6: - if not self._directional_observed: - self._directional_observed = True - self._blocked.append(point) - else: - self._blocked = [] - self._localize_steps += 1 - - self._clear_failed = False - cert = clear_certificate(new_region, self._channel, self.config) \ - if new_region is not None else None - observation = self._observe() - if cert is not None and cert.certified: - return self._try_clear(cert, observation=observation, - reward=reward) - if self.problem == 4 and (len(self._blocked) >= self.config.search.max_blocked_probes - or self._localize_steps >= self.config.search.max_localize_steps): - return self._fallback_clear(observation=observation, reward=reward) - if self._steps >= self.max_episode_steps: - return self._finish(observation=observation, reward=reward - 100.0, - terminated=False, truncated=True) - return observation, reward, False, False, self._info() - - def _fallback_clear(self, *, observation, reward: float): - """问题4后备清除语义：遮挡梯子用尽后按 20m 网格覆盖逐格 /clear。 - - 与 DirectionalExtension._fallback_clear 同口径；训练环境借此让策略 - 学到"梯子耗尽的代价"，运行闭环仍由原后备清除函数执行。 - """ - region = self._belief.candidate_set(self._channel) - cells = fallback_clear_cells(region.polygon, self.config) - self._fallback_attempts = 0 - if len(cells) > self.config.search.fallback_max_cells: - return self._finish(observation=observation, reward=reward - 100.0, - terminated=False, truncated=True) - attempts = 0 - while cells: - cell = min(cells, key=lambda c: ( - math.dist(self._client.state.position, c), c)) - cells.remove(cell) - attempts += 1 - old_virtual = self._client.state.virtual_time_s - outcome = self._client.clear(cell, self._channel) - reward += -(self._client.state.virtual_time_s - old_virtual) / 100.0 - if outcome.result == CLEAR_SUCCESS: - self._belief.mark_cleared(self._channel) - reward += 20.0 - self._fallback_attempts = attempts - return self._finish(observation=observation, reward=reward, - terminated=True, truncated=False) - self._fallback_attempts = attempts - return self._finish(observation=observation, reward=reward - 100.0, - terminated=True, truncated=False) - - def _try_clear(self, cert, *, observation, reward: float = 0.0): - old_virtual = self._client.state.virtual_time_s - outcome = self._client.clear(cert.center, self._channel) - cost_s = self._client.state.virtual_time_s - old_virtual - reward += -cost_s / 100.0 - if outcome.result == CLEAR_SUCCESS: - self._belief.mark_cleared(self._channel) - reward += 20.0 - return self._finish(observation=observation, reward=reward, - terminated=True, truncated=False) - self._clear_failed = True - reward -= 1.0 - if self._steps >= self.max_episode_steps: - return self._finish(observation=observation, reward=reward, - terminated=False, truncated=True) - return observation, reward, False, False, self._info() - - def _finish(self, *, observation, reward, terminated, truncated): - self._terminated = bool(terminated) - self._truncated = bool(truncated) - info = self._info() - info["cleared"] = (self._belief.channels[self._channel].status.value - == "cleared") - self.close() - return observation, float(reward), self._terminated, self._truncated, info - - def close(self): - if (self._client is not None and self._client.state.entered - and not self._client.state.exited): - self._client.exit() diff --git a/src/mcm26b/rl/features.py b/src/mcm26b/rl/features.py deleted file mode 100644 index e30ed18..0000000 --- a/src/mcm26b/rl/features.py +++ /dev/null @@ -1,209 +0,0 @@ -"""RL 状态特征与候选动作生成（问题3/4 共用，运行时与训练环境同源）。 - -设计目标：把"下一检测点去哪"这一连续决策转化为固定顺序的离散动作， -让 PPO 策略在 GPU 上学习，同时不触碰闭环的保守性证明。候选集合始终由 -既有几何规则生成（保证接收盘、对称交会、定向遮挡梯子），RL 只负责在 -这些物理可行候选中排序；证书、覆盖证明与后备清除仍由 planner 层负责。 - -观测维度与候选编号一旦训练即成为模型接口，不得随意重排： -- 全局特征 41 维（含频道 one-hot 20 维）； -- 每个候选 6 维（相对位移、移动耗时、到 MEC 圆心距离、方位、保证接收标记）， - 共 60 × 6 = 360 维； -- 总观测 401 维。 -""" - -from __future__ import annotations - -import math - -import numpy as np - -from ..config import ProblemConfig, Vec2 -from ..planner import safe_disk_radius -from ._constants import GLOBAL_DIM, LADDER_SLOTS, NUM_ACTIONS, OBS_DIM, PER_ACTION_DIM - -__all__ = [ - "GLOBAL_DIM", - "LADDER_SLOTS", - "NUM_ACTIONS", - "OBS_DIM", - "PER_ACTION_DIM", - "candidate_actions", - "build_observation", - "empty_observation", - "action_mask", -] - - -# 候选编号布局（训练/推理共享）： -# 0 保证接收盘圆心（或宽候选集 MEC 圆心） -# 1..36 保证接收盘 3 环 × 12 方向 -# 37..51 相对"机器人->MEC 圆心"来向的对称交会候选 3 环 × 5 偏角 -# 52..59 定向遮挡环探梯子 8 个方向（问题3也保留，作为宽环探索） -_RING_RATIOS = (0.4, 0.75, 1.0) -_RING_ANGLES = 12 -_GEO_OFFSETS = (-math.pi / 3, -math.pi / 6, 0.0, math.pi / 6, math.pi / 3) - - -def _probe_radius(region_radius_m: float, safe_m: float) -> float: - """环候选的基准半径：有保证接收盘用盘半径，否则退化为 MEC 半径。""" - if safe_m >= 0.0: - return safe_m - return max(region_radius_m, 10.0) - - -def candidate_actions(region, position: Vec2, config: ProblemConfig, - blocked: list[Vec2] | None = None) -> list[Vec2]: - """生成固定顺序的 60 个候选检测点。 - - 参数 region 为 belief.CandidateRegion（任意凸候选集）；blocked 是 - 定向遮挡记录（问题4），问题3 传空列表即可。返回顺序与模型训练时一致。 - """ - center, radius = region.mec() - safe = safe_disk_radius(region, config) - base_radius = _probe_radius(radius, safe) - # 半径恰为0时也保留固定编号，重复位置由action_mask屏蔽。 - points = [tuple(center)] - for fraction in _RING_RATIOS: - for k in range(_RING_ANGLES): - angle = (k + 0.5) * 2 * math.pi / _RING_ANGLES - points.append((center[0] + base_radius*fraction*math.cos(angle), - center[1] + base_radius*fraction*math.sin(angle))) - assert len(points) == 37 - - base_angle = math.atan2(position[1] - center[1], - position[0] - center[0]) - for fraction in _RING_RATIOS: - for offset in _GEO_OFFSETS: - angle = base_angle + offset - points.append((center[0] + base_radius * fraction * math.cos(angle), - center[1] + base_radius * fraction * math.sin(angle))) - - blocked = list(blocked or []) - first = blocked[0] if blocked else None - anchor = 0.0 - if first is not None: - anchor = math.atan2(first[1] - center[1], first[0] - center[0]) - ladder_radius = max(radius, 20.0) - for slot in range(LADDER_SLOTS): - angle = anchor + slot * math.pi / 4 - points.append((center[0] + ladder_radius * math.cos(angle), - center[1] + ladder_radius * math.sin(angle))) - assert len(points) == NUM_ACTIONS - return points - - -def action_mask(region, position: Vec2, config: ProblemConfig, problem: int, - blocked=None, belief=None) -> np.ndarray: - """只依据可见观测屏蔽同地重测、重复候选、越界点。 - - Q3有距离保证盘时屏蔽盘外探点；Q4允许盘外探点，但绝不把距离保证 - 当作朝向保证。训练与推理共用此函数，不读取源位置或朝向。 - """ - def key(p): - return (round(p[0]*2), round(p[1]*2)) - visited = {key(r['position']) for r in getattr(belief, 'measurements', [])} - seen = set() - center, _ = region.mec() - safe = safe_disk_radius(region, config) - result = [] - for p in candidate_actions(region, position, config, blocked): - tag = key(p) - valid = (tag not in visited and tag not in seen - and all(math.isfinite(v) and abs(v) <= config.field.coord_abs_limit_m for v in p) - and (problem != 3 or safe < 0 or math.dist(p, center) <= safe + 1e-6)) - result.append(valid) - if valid: - seen.add(tag) - if not any(result): - # 退化几何无新候选时保留一个合法动作，由步数/预算边界收尾。 - result[0] = True - return np.asarray(result, dtype=bool) - - -def _last_measurement_features(belief) -> tuple[tuple[float, float, float], - tuple[float, float]]: - """末次读数的 one-hot + 方向角 sin/cos（无读数取零）。""" - readings = getattr(belief, "measurements", None) or [] - if not readings: - return (1.0, 0.0, 0.0), (0.0, 0.0) - last = readings[-1] - result = last.get("result") - no_signal = 1.0 if result == "no_signal" else 0.0 - direction = 1.0 if result == "direction" else 0.0 - near = 1.0 if result == "near" else 0.0 - svd = last.get("svd_deg") - if direction and svd is not None: - radians = math.radians(float(svd)) - return (no_signal, direction, near), (math.sin(radians), math.cos(radians)) - return (no_signal, direction, near), (0.0, 0.0) - - -def build_observation(region, position: Vec2, channel: int, config: ProblemConfig, - problem: int, blocked: list[Vec2] | None = None, - belief=None, virtual_time_s: float = 0.0, - receiver_channel: int | None = None) -> np.ndarray: - """构造 PPO 观测向量（shape=(401,)，与训练环境逐位一致）。""" - field, search = config.field, config.search - center, radius = region.mec() - safe = safe_disk_radius(region, config) - readings, svd_pair = _last_measurement_features(belief) - xs = [p[0] for p in region.polygon] - ys = [p[1] for p in region.polygon] - - global_features = [ - position[0] / 2000.0, - position[1] / 2000.0, - ((channel if receiver_channel is None else receiver_channel) - field.channel_min) - / max(field.channel_max - field.channel_min, 1), - center[0] / 2000.0, - center[1] / 2000.0, - radius / 2000.0, - max(-1.0, min(1.0, safe / 1000.0)), - min(len(getattr(belief, "measurements", []) if belief is not None else []), - 12) / 12.0, - min(float(virtual_time_s or 0.0), field.max_virtual_duration_s) - / field.max_virtual_duration_s, - # 证书信息由运行时的 clear_certificate 判定传入；训练环境用 region.mec - # 的保守近似：radius + slack <= 20 - margin 即可清除。 - 1.0 if radius + region.slack_m <= field.clear_radius_m - search.clear_margin_m else 0.0, - float(problem == 4), - min(len(blocked or []) / max(search.max_blocked_probes, 1), 1.0), - (blocked[0][0] / 2000.0) if blocked else 0.0, - (blocked[0][1] / 2000.0) if blocked else 0.0, - readings[0], - readings[1], - readings[2], - svd_pair[0], - svd_pair[1], - (max(xs) - min(xs)) / 4000.0, - (max(ys) - min(ys)) / 4000.0, - ] - one_hot = np.zeros(field.channel_max - field.channel_min + 1, dtype=np.float32) - one_hot[int(channel) - field.channel_min] = 1.0 - global_features.extend(one_hot.tolist()) - assert len(global_features) == GLOBAL_DIM - - action_features: list[float] = [] - for point in candidate_actions(region, position, config, blocked): - dx, dy = point[0] - position[0], point[1] - position[1] - distance = math.hypot(dx, dy) - action_features.extend([ - dx / 2000.0, - dy / 2000.0, - min(distance / (config.cost.speed_mps * 1500.0), 2.0), # 移动耗时 / 1500s - math.hypot(point[0] - center[0], point[1] - center[1]) / 2000.0, - (math.degrees(math.atan2(dy, dx)) % 360.0) / 360.0, - 1.0 if safe >= 0 and math.hypot(point[0] - center[0], - point[1] - center[1]) <= safe + 1e-6 else 0.0, - ]) - assert len(action_features) == NUM_ACTIONS * PER_ACTION_DIM - - observation = np.asarray(global_features + action_features, dtype=np.float32) - assert observation.shape == (OBS_DIM,) - return observation - - -def empty_observation() -> np.ndarray: - """依赖缺失时的零观测（仅用于错误提示路径，不进入训练数据）。""" - return np.zeros(OBS_DIM, dtype=np.float32) diff --git a/src/mcm26b/rl/policy.py b/src/mcm26b/rl/policy.py deleted file mode 100644 index 5a5ca97..0000000 --- a/src/mcm26b/rl/policy.py +++ /dev/null @@ -1,95 +0,0 @@ -"""RL 选点策略加载与推理（运行时接入 planner 的唯一入口）。 - -运行闭环对依赖保持惰性：只有显式选择 ``--strategy rl`` / ``--q4-strategy rl`` -时才导入 stable_baselines3/torch。模型文件由 scripts/train_rl.py 生成， -观测/动作布局与训练环境共享 features 模块。 -""" - -from __future__ import annotations - -from pathlib import Path -import json - -from ..config import ProblemConfig, Vec2 -from ._constants import NUM_ACTIONS, OBS_DIM -from .features import action_mask, build_observation, candidate_actions - - -class PointSelectionPolicy: - """加载 PPO 模型，把当前候选集映射到 60 个候选检测点之一。""" - - def __init__(self, model_path: str | Path, *, device: str = "auto"): - path = Path(model_path) - if not path.is_file(): - raise RuntimeError( - f"未找到 RL 模型 {path}。请先安装 rl 依赖组并训练：\n" - f" uv sync --group rl --group dev\n" - f" uv run python scripts/train_rl.py --problem 3 --timesteps 120000\n" - f" uv run python scripts/train_rl.py --problem 4 --timesteps 40000") - try: - from stable_baselines3 import PPO - except ImportError as exc: - raise RuntimeError( - "缺少 RL 运行依赖（stable-baselines3）。请执行 " - "`uv sync --group rl` 后重试。") from exc - if device == "auto": - try: - import torch - device = "cuda" if torch.cuda.is_available() else "cpu" - except ImportError: - device = "cpu" - self.device = device - meta_path = path.with_suffix('.json') - self.metadata = (json.loads(meta_path.read_text(encoding='utf-8')) - if meta_path.exists() else {}) - self.feature_version = self.metadata.get('feature_version', 1) - if self.feature_version not in (1, 2): - raise RuntimeError('不支持的RL特征版本') - self.masked = self.metadata.get('algorithm') == 'maskable_ppo' - if self.masked: - from sb3_contrib import MaskablePPO - if self.metadata.get('feature_version') != 2: - raise RuntimeError('动作屏蔽模型的特征版本不匹配') - loader = MaskablePPO - else: - loader = PPO - self.model = loader.load(str(path), device=device) - if not self.masked and type(self.model.policy).__module__.startswith('sb3_contrib'): - raise RuntimeError('动作屏蔽模型缺少匹配的JSON元数据，不能按普通PPO推理') - try: - obs_dim = self.model.observation_space.shape[0] - action_dim = self.model.action_space.n - except Exception as exc: - raise RuntimeError(f"模型空间与 RL 接口不兼容: {path}") from exc - if obs_dim != OBS_DIM or action_dim != NUM_ACTIONS: - raise RuntimeError( - f"模型接口失配: 期望 obs={OBS_DIM}/act={NUM_ACTIONS}，" - f"实际 obs={obs_dim}/act={action_dim}。请用当前版本重新训练。") - self.path = path - - def select(self, region, position: Vec2, channel: int, - config: ProblemConfig, problem: int, - blocked: list[Vec2] | None = None, belief=None, - virtual_time_s: float = 0.0, - receiver_channel: int | None = None) -> tuple[Vec2, int]: - """返回 (检测点, 候选编号)；确定性 argmax（评估/比赛口径）。""" - if self.metadata.get('problem', problem) != problem: - raise RuntimeError('模型训练问题编号与当前第三/四问不匹配') - observation = build_observation( - region, position, channel, config, problem, blocked, belief, - virtual_time_s=virtual_time_s, - receiver_channel=receiver_channel if self.feature_version >= 2 else None) - kwargs = ({'action_masks': action_mask(region, position, config, problem, - blocked, belief)} - if self.masked else {}) - action, _state = self.model.predict(observation, deterministic=True, **kwargs) - action = int(action) - candidates = candidate_actions(region, position, config, blocked) - return candidates[action], action - - -def default_model_path(problem: int) -> Path: - """各问题的默认模型位置（训练脚本输出口径）。""" - if problem not in (3, 4): - raise ValueError("problem 必须是 3 或 4") - return Path("output/rl") / f"q{problem}_ppo.zip" diff --git a/src/mcm26b/training_ui/__init__.py b/src/mcm26b/training_ui/__init__.py deleted file mode 100644 index ea762b2..0000000 --- a/src/mcm26b/training_ui/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -"""模型训练前端：本地回环 Web UI + 训练子进程监控。 - -启动： - python -m mcm26b.training_ui --ui-port 8036 -或： - python scripts/train_hrl_ui.py --ui-port 8036 - -子命令： - python -m mcm26b.training_ui --print-schema # 配置契约 - python -m mcm26b.training_ui --print-preview # 离线示例快照 -""" -from .server import TrainingUI, TrainingJob, run_training_ui -from .schema import config_schema - -__all__ = ["TrainingUI", "TrainingJob", "run_training_ui", "config_schema"] diff --git a/src/mcm26b/training_ui/__main__.py b/src/mcm26b/training_ui/__main__.py deleted file mode 100644 index 2a16926..0000000 --- a/src/mcm26b/training_ui/__main__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .server import run_training_ui - -if __name__ == "__main__": - raise SystemExit(run_training_ui()) diff --git a/src/mcm26b/training_ui/analysis.py b/src/mcm26b/training_ui/analysis.py deleted file mode 100644 index a48ff28..0000000 --- a/src/mcm26b/training_ui/analysis.py +++ /dev/null @@ -1,397 +0,0 @@ -"""训练快照的纯函数派生：曲线统计、收敛判读、进程视图。 - -从 ``server.py`` 拆出来的两个理由： - -1. 断开 ``demo`` ↔ ``server`` 的循环导入——离线预览（``demo.py``）只想复用 - 这两个纯函数，没必要反向 import 整个 HTTP 服务器模块； -2. 让"数值判读"可以脱离 HTTP 与子进程单独测试。 - -本模块只依赖标准库，不做任何 IO、不持有任何状态。 -""" - -from __future__ import annotations - -import math -import re -import subprocess -import time -from typing import Any - -__all__ = ["MAX_LOG_LINES", "MAX_METRICS", "finite_float", - "summarize_convergence", "describe_process", "process_pid"] - -MAX_LOG_LINES = 4000 -MAX_METRICS = 5000 - -_NUM_RE = re.compile(r"^[+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?$") - - -def finite_float(value: Any) -> float | None: - """把任意输入转成有限浮点数；NaN / ±inf / 非数字一律返回 None。 - - 旧实现用 ``value != value`` 判 NaN，pylint 报 ``comparison-with-itself``； - 这里用 ``math.isfinite`` 表达同一语义，意图更直白。 - """ - if value is None or isinstance(value, bool): - return None - if isinstance(value, (int, float)): - value = float(value) - if not math.isfinite(value): - return None - return value - text = str(value).strip() - if not _NUM_RE.match(text): - return None - try: - return float(text) - except ValueError: - return None - - -def _series(metrics: list[dict[str, Any]], key: str) -> list[tuple[float, float]]: - points: list[tuple[float, float]] = [] - for point in metrics: - value = finite_float(point.get(key)) - steps = finite_float(point.get("timesteps")) or finite_float( - point.get("total_timesteps")) - if value is None: - continue - points.append((steps if steps is not None else float(len(points) + 1), value)) - return points - - -def _last_value(points: list[tuple[float, float]]) -> float | None: - return points[-1][1] if points else None - - -def _spread(points: list[tuple[float, float]], window: int = 10 - ) -> tuple[float, float] | None: - """最近 window 个点的 (极差, 起点绝对值)，用于判断"是否还在动"。""" - sample = points[-window:] - if len(sample) < 3: - return None - ys = [item[1] for item in sample] - return max(ys) - min(ys), abs(ys[0]) - - -def _cv(points: list[tuple[float, float]], window: int = 10) -> float | None: - """最近 window 个点的变异系数（标准差/|均值|），越小越平稳。""" - sample = [item[1] for item in points[-window:]] - if len(sample) < 3: - return None - mean = sum(sample) / len(sample) - if abs(mean) < 1e-12: - return None - var = sum((value - mean) ** 2 for value in sample) / len(sample) - return (var ** 0.5) / abs(mean) - - -def _slope(points: list[tuple[float, float]], window: int = 8) -> float | None: - sample = points[-window:] - if len(sample) < 3: - return None - xs = [item[0] for item in sample] - ys = [item[1] for item in sample] - n = len(xs) - mean_x = sum(xs) / n - mean_y = sum(ys) / n - denom = sum((x - mean_x) ** 2 for x in xs) - if abs(denom) < 1e-12: - return None - return sum((x - mean_x) * (y - mean_y) for x, y in zip(xs, ys)) / denom - - -def _rate_per_1k(reward_points: list[tuple[float, float]], - window: int = 8) -> float | None: - """最近 window 个点的奖励增益换算成"每 1000 环境步"，便于读趋势。""" - slope = _slope(reward_points, window) - if slope is None: - return None - return slope * 1000.0 - - -def _fmt_num(value: float | None, digits: int = 4) -> str: - if value is None: - return "—" - if value != 0 and abs(value) < 1e-3: - return f"{value:.3e}" - return f"{value:.{digits}g}" - - -def _flatness(points: list[tuple[float, float]], window: int = 10 - ) -> tuple[float, float] | None: - """窗口内"趋势位移 / 逐点噪声"比值：<1 表示看不出系统性变化。 - - 判定收敛时只看斜率绝对值会把"很慢但持续的提升"误判为平台；这里把窗口 - 对半切，用两半均值之差当趋势位移，用相邻点平均绝对差当噪声尺度。返回 - (比值, 噪声尺度)，两者都小才是真的"又平又稳"。 - """ - sample = points[-window:] - if len(sample) < 4: - return None - ys = [item[1] for item in sample] - half = len(ys) // 2 - shift = abs(sum(ys[half:]) / len(ys[half:]) - sum(ys[:half]) / len(ys[:half])) - noise = sum(abs(b - a) for a, b in zip(ys, ys[1:])) / (len(ys) - 1) - if noise <= 0: - return (0.0, 0.0) if shift <= 0 else (float("inf"), 0.0) - return shift / noise, noise - - -def summarize_convergence(metrics: list[dict[str, Any]], - monitor: list[dict[str, Any]]) -> dict[str, Any]: - """给前端返回可解释的收敛状态与各参数最近值。 - - 这是**启发式判读**，不替代正式学习曲线分析：只依据最近窗口的斜率、 - 极差与变异系数给出"平坦/上升/未上升"的分类，并把同一组数字交给前端 - 展示，便于人工复核。 - """ - reward_points = _series(metrics, "ep_rew_mean") - if not reward_points and monitor: - reward_points = [(float(index), float(row.get("r", 0.0))) - for index, row in enumerate(monitor) - if finite_float(row.get("r")) is not None] - kl_points = _series(metrics, "approx_kl") - ev_points = _series(metrics, "explained_variance") - vf_points = _series(metrics, "value_loss") - pg_points = _series(metrics, "policy_gradient_loss") - entropy_points = _series(metrics, "entropy_loss") - fps_points = _series(metrics, "fps") - - reward_slope = _slope(reward_points) - reward_rate = _rate_per_1k(reward_points) - spread = _spread(reward_points) - cv = _cv(reward_points) - latest_kl = _last_value(kl_points) - latest_ev = _last_value(ev_points) - latest_vf = _last_value(vf_points) - latest_pg = _last_value(pg_points) - latest_entropy = _last_value(entropy_points) - latest_fps = _last_value(fps_points) - latest_timesteps = metrics[-1].get("timesteps") if metrics else None - if latest_timesteps is None and metrics: - latest_timesteps = metrics[-1].get("total_timesteps") - latest_reward = _last_value(reward_points) - latest_len = _last_value(_series(metrics, "ep_len_mean")) - latest_updates = _last_value(_series(metrics, "n_updates")) - window = min(10, max(3, len(reward_points))) - reward_window = [round(value, 6) for _, value in reward_points[-window:]] - best_reward = max((value for _, value in reward_points), default=None) - - notes: list[str] = [] - if not metrics and not monitor: - status, tone = "等待首个 rollout", "gray" - notes.append("训练进程启动后先输出环境初始化与首个 rollout 的进度行。") - elif not metrics: - status, tone = "已收到回合奖励，等待进度行", "warn" - notes.append("Monitor CSV 已有回合记录，但尚未收到 --progress-json 进度行；" - "首个 rollout 结束后会出现完整指标。") - elif len(metrics) < 3: - status, tone = "预热阶段", "warn" - notes.append("已收到初始指标，需要更多 rollout 才能判断趋势。") - else: - tone, status = "ok", "收敛趋势良好" - # 平台判据（三条同时成立）：对半窗口看不出位移、窗口内波动小、 - # 逐点噪声本身也小。慢速但持续的提升会因第一条被排除。 - flat = _flatness(reward_points) - plateau = False - if flat is not None and cv is not None and latest_reward is not None: - flat_ratio, flat_noise = flat - scale = max(abs(latest_reward), 1e-9) - plateau = (flat_ratio <= 1.0 and cv <= 0.15 - and flat_noise <= 0.02 * scale - and (spread is None or spread[1] < 1e-12 - or spread[0] <= 0.15 * spread[1])) - if latest_kl is not None and latest_kl > 0.05: - status, tone = "KL 偏大，策略更新可能过猛", "bad" - notes.append(f"approx_kl={latest_kl:.4f} > 0.05") - elif latest_ev is not None and latest_ev < 0.0: - status, tone = "价值函数尚未拟合", "warn" - notes.append(f"explained_variance={latest_ev:.3f} < 0") - elif reward_slope is not None and reward_slope < 0: - status, tone = "奖励未形成上升趋势", "warn" - notes.append(f"最近 {window} 点奖励斜率为负（{_fmt_num(reward_slope)}/step）") - elif plateau: - status, tone = "已趋于平稳（疑似收敛）", "ok" - notes.append("窗口内对半位移小于逐点噪声，且波动很小" - f"（变异系数 {_fmt_num(cv, 3)}）") - else: - notes.append(f"最近 {window} 点奖励斜率为正（{_fmt_num(reward_slope)}/step）") - if latest_kl is not None and latest_kl <= 0.02: - notes.append(f"approx_kl={latest_kl:.4f} 较稳定") - if latest_ev is not None: - notes.append(f"explained_variance={latest_ev:.3f}") - if cv is not None: - notes.append(f"奖励变异系数={_fmt_num(cv, 3)}") - if latest_entropy is not None and abs(latest_entropy) < 1e-3: - notes.append("策略熵接近 0，探索可能过早停止") - return { - "status": status, - "tone": tone, - "notes": notes, - "window": window, - "latest_timesteps": latest_timesteps, - "reward_slope": reward_slope, - "reward_rate_per_1k": reward_rate, - "reward_cv": cv, - "reward_spread": spread[0] if spread else None, - "reward_window": reward_window, - "best_reward": best_reward, - "latest_reward": latest_reward, - "latest_ep_len": latest_len, - "latest_updates": latest_updates, - "latest_kl": latest_kl, - "latest_explained_variance": latest_ev, - "latest_value_loss": latest_vf, - "latest_policy_loss": latest_pg, - "latest_entropy_loss": latest_entropy, - "latest_fps": latest_fps, - "reward_points": len(reward_points), - } - - -# 快照状态 -> 界面展示用的进程语义（阶段名称、色调、是否终态） -_STATUS_VIEW = { - "created": ("已创建", "gray", False, "命令已构造，等待 spawn"), - "running": ("运行中", "ok", False, "子进程持有控制台输出与独立进程组"), - "paused": ("已暂停", "warn", False, "进程存活但停在 step 边界；点「继续」从暂停处恢复"), - "stopping": ("停止中", "warn", False, "已请求优雅停止，等待子进程保存并退出"), - "stopped": ("已手动停止", "warn", True, "收到停止请求后退出；脚本会在 step 边界保存模型与元数据"), - "exited": ("正常结束", "ok", True, "退出码 0：训练与评估流程走完"), - "failed": ("异常结束", "bad", True, "非零退出码或启动失败，见日志尾部"), -} - -_PHASES = [ - ("created", "构造命令"), - ("spawn", "启动子进程"), - ("rollout", "采集 rollout"), - ("update", "PPO 更新"), - ("evaluate", "评估 / 保存"), - ("done", "收敛归档"), -] - -# 状态 -> 当前所处的阶段节点；节点之前为已完成，之后为未开始。 -_STATUS_PHASE = { - "created": "created", - "running": "update", - "paused": "update", - "stopping": "update", - "stopped": "done", - "exited": "done", - "failed": "done", -} - - -def process_pid(process: subprocess.Popen | None) -> int | None: - """已退出进程的 pid 访问会抛 OSError；这里统一吞掉，只用于展示。""" - if process is None: - return None - try: - return int(process.pid) - except (OSError, ValueError): - return None - - -def describe_process(snapshot: dict[str, Any]) -> dict[str, Any]: - """把 snapshot 派生为界面直接可渲染的进程对象。 - - 包含：状态语义词、进度（步/百分比）、墙钟与剩余时间估计、最近吞吐、 - 进程时间线节点。数字都取自 snapshot，前端不再重复累加，避免两个刷新 - 频率不同的计数器互相打架。 - """ - status = str(snapshot.get("status") or "created") - label, tone, terminal, hint = _STATUS_VIEW.get( - status, (status, "gray", False, "")) - config = snapshot.get("config") or {} - order = [key for key, _ in _PHASES] - current = _STATUS_PHASE.get(status, "created") - timeline = [] - for key, title in _PHASES: - index = order.index(key) - if index < order.index(current): - state = "done" - elif index == order.index(current): - state = "active" - else: - state = "pending" - timeline.append({"key": key, "label": title, "state": state}) - - metrics = snapshot.get("metrics") or [] - latest: dict[str, Any] = metrics[-1] if metrics else {} - convergence = snapshot.get("convergence") or {} - total_steps = finite_float(config.get("timesteps")) or 0.0 - # SB3 表格行可能没有 steps 字段，因此进度取"所有进度点的最大步数"， - # 而不是只看最后一个点——否则末行恰好缺字段时进度会回退到 0。 - done_steps = 0.0 - for point in metrics: - value = (finite_float(point.get("timesteps")) - or finite_float(point.get("total_timesteps"))) - if value is not None and value > done_steps: - done_steps = value - if done_steps == 0.0: - done_steps = finite_float(convergence.get("latest_timesteps")) or 0.0 - # init/resume（课程迁移）时 num_timesteps 从上一阶段继续，进度必须按本次 - # 增量算；start_timesteps 由训练脚本的进度 JSON 提供，缺省按 0 处理。 - start_steps = 0.0 - for point in metrics: - value = finite_float(point.get("start_timesteps")) - if value is not None: - start_steps = value - break - trained_steps = max(0.0, done_steps - start_steps) - percent = (100.0 * trained_steps / total_steps) if total_steps > 0 else None - if percent is not None: - percent = max(0.0, min(100.0, percent)) - elapsed = float(snapshot.get("elapsed_s") or 0.0) - rate = _latest_rate(metrics) - remaining: float | None = None - eta_ts: float | None = None - if rate and rate > 0 and total_steps > 0 and trained_steps < total_steps: - remaining = (total_steps - trained_steps) / rate - eta_ts = float(snapshot.get("finished_at") or time.time()) + remaining - if not terminal and remaining is None: - cap = finite_float(config.get("max_wall_s")) or 0.0 - if cap > 0: - remaining = max(0.0, cap - elapsed) - return { - "label": label, - "tone": tone, - "terminal": terminal, - "hint": hint, - "phase": current, - "timeline": timeline, - "percent": percent, - "done_steps": done_steps, - "start_steps": start_steps, - "trained_steps": trained_steps, - "total_steps": total_steps, - "elapsed_s": round(elapsed, 3), - "rate_per_s": rate, - "remaining_s": remaining, - "eta_epoch_s": eta_ts, - "log_total": snapshot.get("log_total"), - "log_capacity": MAX_LOG_LINES, - "metric_total": snapshot.get("metric_total"), - "latest": latest, - } - - -def _latest_rate(metrics: list[dict[str, Any]], window: int = 6) -> float | None: - """最近若干个**带 fps 的**进度点的平均吞吐（步/秒），用于剩余时间估计。 - - progress JSON 行不含 fps，只有 SB3 表格行有；若只看最后 window 个点， - 恰好落在若干 JSON 行之间时会得到 None，剩余时间就会退化成墙钟上限的 - 粗糙估计。这里向前扫描直到拿到至少 1 个 fps（最多看 3*window 个点）。 - """ - if not metrics: - return None - usable: list[float] = [] - for point in reversed(metrics[-window * 3:]): - value = finite_float(point.get("fps")) - if value and value > 0: - usable.append(value) - if len(usable) >= window: - break - if not usable: - return None - return sum(usable) / len(usable) diff --git a/src/mcm26b/training_ui/demo.py b/src/mcm26b/training_ui/demo.py deleted file mode 100644 index 914777d..0000000 --- a/src/mcm26b/training_ui/demo.py +++ /dev/null @@ -1,223 +0,0 @@ -"""训练前端示例快照：把一段"看起来像真的" PPO 训练曲线合成为合法快照。 - -用途有两个，都不涉及真实训练： -1. ``GET /api/train/state?preview=1`` 让界面在没有训练进程时也能验证渲染 - （曲线、指标卡、进程时间线、日志着色、逐回合表）； -2. 单元测试可以断言前端的收敛判读分支（上升/平坦/KL 过猛）各有稳定输入。 - -曲线是解析函数 + 固定种子的确定性噪声，不含任何真实评测数据。 -""" - -from __future__ import annotations - -import math -import random -import time -from typing import Any - -from .analysis import describe_process, summarize_convergence - -__all__ = ["DEMO_CONFIG", "DEMO_SCENARIOS", "demo_metrics", "demo_monitor", - "demo_log_lines", "demo_snapshot"] - -DEMO_SCENARIOS = ("rising", "plateau", "kl-spike") - -DEMO_CONFIG: dict[str, Any] = { - "problem": 3, "stage": 2, "timesteps": 12000, "seed": 320001, - "envs": 4, "vec_env": "subproc", "device": "mps", - "config_profile": "production", "reward_profile": "time", - "model": "output/hrl/ui/demo/q3.zip", "init": None, "resume": False, - "eval_episodes": 2, "rollout_steps": 256, "batch_size": 256, - "n_epochs": 10, "max_wall_s": 1200.0, "local_selector": "geometry", - "local_model": None, "verbose": 1, -} - -DEMO_COMMAND = [ - "python", "scripts/train_hrl_high.py", "--problem", "3", "--stage", "2", - "--timesteps", "12000", "--seed", "320001", "--envs", "4", - "--vec-env", "subproc", "--device", "mps", "--config-profile", "production", - "--reward-profile", "time", "--model", "output/hrl/ui/demo/q3.zip", - "--eval-episodes", "2", "--rollout-steps", "256", "--batch-size", "256", - "--n-epochs", "10", "--max-wall-s", "1200.0", "--local-selector", - "geometry", "--progress-json", "--verbose", "1", -] - -DEMO_STEPS = 48 - - -def _finite(value: float) -> float | None: - if not math.isfinite(value): - return None - return round(value, 8) - - -def demo_metrics(count: int = DEMO_STEPS, steps: int = 12000, *, - seed: int = 7, scenario: str = "rising", - total_steps: int | None = None, fps: float = 420.0 - ) -> list[dict[str, Any]]: - """合成 count 个进度点，横轴覆盖到 ``total_steps`` 的 ``count/DEMO_STEPS``。 - - 场景（用于验证收敛判读的各个分支，不含真实数据）： - rising 奖励对数饱和上升、KL 正常 → "收敛趋势良好" - plateau 奖励已进入平台期 → "已趋于平稳（疑似收敛）" - kl-spike 末段近似 KL 冲高 → "KL 偏大，策略更新可能过猛" - """ - if scenario not in DEMO_SCENARIOS: - raise ValueError(f"未知示例场景: {scenario}") - total = int(total_steps if total_steps is not None else steps) - rng = random.Random(seed) - points: list[dict[str, Any]] = [] - per = total / DEMO_STEPS - ceiling = 42.0 - span = min(1.0, max(1e-6, count / float(DEMO_STEPS))) - for index in range(1, count + 1): - t = (index / float(DEMO_STEPS)) / span # 折到整段训练的进度比例 - steps_now = int(round(index * per)) - if scenario == "plateau": - reward = 41.5 + rng.uniform(-0.35, 0.35) - value_loss_curve = 3.6 + rng.uniform(-0.2, 0.2) - approx_kl = max(0.001, 0.004 + rng.uniform(-0.001, 0.001)) - explained = min(0.95, 0.88 + rng.uniform(-0.02, 0.02)) - elif scenario == "kl-spike": - reward = ceiling * (1.0 - math.exp(-3.1 * t)) + rng.uniform(-1.2, 1.2) - value_loss_curve = 62.0 * math.exp(-2.4 * t) + 3.4 - base_kl = 0.031 * math.exp(-1.7 * t) - approx_kl = base_kl + (0.075 if t > 0.8 else 0.0) + rng.uniform(-0.003, 0.003) - explained = -0.09 + 1.05 * (1.0 - math.exp(-2.2 * t)) - else: - noise = rng.uniform(-1.4, 1.4) * (1.0 - 0.6 * t) - reward = ceiling * (1.0 - math.exp(-3.1 * t)) + noise - value_loss_curve = 62.0 * math.exp(-2.4 * t) + 3.4 - approx_kl = 0.031 * math.exp(-1.7 * t) - explained = -0.09 + 1.05 * (1.0 - math.exp(-2.2 * t)) - value_loss = value_loss_curve + rng.uniform(-0.5, 0.5) - policy_loss = -0.021 + 0.006 * math.sin(index / 4.0) + rng.uniform(-0.002, 0.002) - entropy = 1.35 - 0.55 * (t if scenario != "plateau" else 0.9) \ - + rng.uniform(-0.02, 0.02) - points.append({ - "timesteps": steps_now, - "n_updates": index * 12, - "ep_rew_mean": _finite(reward), - "ep_len_mean": _finite(24.0 + 21.0 * t + rng.uniform(-1.5, 1.5)), - "value_loss": _finite(value_loss), - "policy_gradient_loss": _finite(policy_loss), - "approx_kl": _finite(max(0.0008, approx_kl + rng.uniform(-0.002, 0.002))), - "entropy_loss": _finite(entropy), - "explained_variance": _finite(min(0.97, explained - + rng.uniform(-0.03, 0.03))), - # 吞吐尽量接近真实量级（envs=4 时约 400+ 步/秒），否则剩余时间 - # 估计会给出荒谬的 ETA - "fps": _finite(max(30.0, fps * (0.92 + 0.16 * t) - + rng.uniform(-12.0, 12.0))), - }) - return points - - -def demo_monitor(count: int = 60, seed: int = 11) -> list[dict[str, Any]]: - rng = random.Random(seed) - rows = [] - clock = 0.0 - for index in range(count): - t = index / max(1, count - 1) - clock += rng.uniform(4.0, 14.0) - rows.append({ - "r": _finite(38.0 * (1.0 - math.exp(-3.0 * t)) + rng.uniform(-3.0, 3.0)), - "l": float(rng.randint(28, 74)), - "t": round(clock, 4), - }) - return rows - - -def demo_log_lines() -> list[str]: - """一段包含各类着色分支的日志：SB3 表格、进度 JSON、警告与收尾。""" - lines = [ - "=== 训练前端离线预览：以下为示例快照，非真实训练输出 ===", - "使用设备: mps", - "创建 4 个 DummyVecEnv/SubprocVecEnv 环境 (stage=2, problem=3)", - "Monitor CSV: output/hrl/ui/demo/high_curve_q3_env0.monitor.csv", - "-----------------------------", - "| rollout/ | |", - "| ep_len_mean | 31.4 |", - "| ep_rew_mean | 12.83 |", - "| time/ | |", - "| fps | 412 |", - "| total_timesteps | 512 |", - "-----------------------------", - ] - for index in range(1, 9): - t = index / 8.0 - lines.append( - '{"event": "progress", "timesteps": %d, "n_updates": %d, ' - '"ep_rew_mean": %.3f, "ep_len_mean": %.2f, "value_loss": %.4f, ' - '"policy_gradient_loss": %.5f, "approx_kl": %.5f, ' - '"entropy_loss": %.4f, "explained_variance": %.4f, "fps": %.1f}' - % (index * 1500, index * 12, 42.0 * (1 - math.exp(-3.1 * t)), - 24 + 21 * t, 62.0 * math.exp(-2.4 * t) + 3.4, -0.021 + 0.006 * t, - 0.031 * math.exp(-1.7 * t), 1.35 - 0.55 * t, - -0.09 + 1.05 * (1 - math.exp(-2.2 * t)), 390 + 70 * t) - ) - lines += [ - "UserWarning: 示例警告行，用于验证警告着色", - "确定性评估：2/2 成功，accounted=20，无异常", - "模型已保存: output/hrl/ui/demo/q3.zip", - "元数据已写入: output/hrl/ui/demo/q3.json", - ] - return lines - - -def demo_snapshot(*, log_tail: int = 400, scenario: str = "rising", - done_ratio: float = 0.62, fps: float = 420.0) -> dict[str, Any]: - """合成一次"跑到中途"的快照（键与 TrainingJob.snapshot 一致）。 - - ``done_ratio`` 控制已完成的步数比例，用来让预览更接近真实运行中状态 - （而不是刚好 100%）；``scenario`` 选择收敛判读分支；``fps`` 决定剩余 - 时间估计的吞吐量级。 - """ - now = time.time() - ratio = min(1.0, max(0.05, float(done_ratio))) - metrics = demo_metrics(count=max(6, int(round(DEMO_STEPS * ratio))), - total_steps=int(DEMO_CONFIG["timesteps"]), - scenario=scenario, fps=fps) - monitor = demo_monitor(count=max(6, int(round(60 * ratio)))) - config = dict(DEMO_CONFIG) - lines = demo_log_lines() - logs = (lines * 12)[-max(1, int(log_tail)):] - status = "running" - payload: dict[str, Any] = { - "job_id": "job-preview", - "status": status, - "pid": 42424, - "created_at": now - 186.0, - "started_at": now - 184.0, - "finished_at": None, - "elapsed_s": 184.0, - "returncode": None, - "error": None, - "forced_kill": False, - "command": list(DEMO_COMMAND), - "model": str(config["model"]), - "config": config, - "log_lines": logs, - "log_total": len(logs) * 3, - "metrics": metrics, - "metric_offset": 0, - "metric_end": len(metrics), - "metric_total": len(metrics), - "monitor": monitor, - "recovery": { - "can_resume": True, - "model": str(config["model"]), - "timesteps": int(DEMO_STEPS * ratio * 12), - "reason": "recovery", - "updated_unix_s": now - 24.0, - "source": "resume.json", - "hint_file": str(config["model"]).rsplit("/", 1)[0] + "/resume.json", - "candidates": 3, - "preview": True, - }, - "preview": True, - "preview_scenario": scenario, - } - payload["convergence"] = summarize_convergence(metrics, monitor) - payload["process"] = describe_process(payload) - return payload diff --git a/src/mcm26b/training_ui/schema.py b/src/mcm26b/training_ui/schema.py deleted file mode 100644 index 05940e7..0000000 --- a/src/mcm26b/training_ui/schema.py +++ /dev/null @@ -1,328 +0,0 @@ -"""训练前端配置契约：字段元数据、分组、预设与指标字典。 - -为什么单独成模块：前端表单的取值域、单位、分组与后端 ``TrainConfig`` 的 -校验规则必须来自同一处，否则界面会允许后端拒绝的输入（或反之）。这里只 -声明"可展示"的元数据，真正的合法性判定仍由 :class:`TrainConfig.validate` -单点负责——前端只做提前提示，不复制规则。 - -指标字典 ``METRIC_CATALOG`` 同样是唯一出处：``server.summarize_convergence`` -用它取数，``/api/train/schema`` 把它交给前端画图，避免两边各写一份键名。 -""" - -from __future__ import annotations - -from typing import Any - -__all__ = [ - "CONFIG_FIELDS", "CONFIG_GROUPS", "PRESETS", "METRIC_CATALOG", - "config_schema", "preset_payloads", "metric_keys", -] - -# 字段元数据： -# kind number|integer|text|select|bool -# 其余键 前端直接使用（label/unit/group/choices/min/max/step/help/advanced） -CONFIG_FIELDS: list[dict[str, Any]] = [ - # ---------------------------------------------------------- 环境与课程 - { - "key": "problem", "kind": "select", "group": "env", - "label": "问题", "default": 3, "choices": [ - {"value": 3, "label": "问题 3（全向源）"}, - {"value": 4, "label": "问题 4（含定向源）"}, - ], - "help": "决定场景源类型与末段课程配置。", - }, - { - "key": "stage", "kind": "select", "group": "env", - "label": "课程阶段", "default": 1, "choices": [ - {"value": 1, "label": "1 · 单源 / 5 频道 / 无噪"}, - {"value": 2, "label": "2 · 3 源 / 10 频道 / ±0.5°"}, - {"value": 3, "label": "3 · 8 源 / 16 频道 / ±1°"}, - {"value": 4, "label": "4 · 10–16 源 / 20 频道 / ±1°"}, - ], - "help": "课程递进；跨阶段续训用下方 --init 迁移权重。", - }, - { - "key": "config_profile", "kind": "select", "group": "env", - "label": "配置档", "default": "production", "choices": [ - {"value": "production", "label": "production（生产保真）"}, - {"value": "coarse-pretrain", "label": "coarse-pretrain（快速粗训）"}, - ], - "help": "coarse-pretrain 显式映射为 fast 评分配置并写入模型元数据。", - }, - { - "key": "reward_profile", "kind": "select", "group": "env", - "label": "奖励口径", "default": "time", "choices": [ - {"value": "time", "label": "time（P0.4 纯时间目标）"}, - {"value": "prompt", "label": "prompt（§5.2 原始塑形）"}, - ], - "help": "对照实验用；口径固化进模型元数据，不静默替换。", - }, - { - "key": "seed", "kind": "integer", "group": "env", - "label": "随机种子", "default": 320001, "min": 0, "max": 2_147_483_647, - "help": "不同种子用于多次重复训练与方差估计。", - }, - # ---------------------------------------------------------- PPO 超参 - { - "key": "rollout_steps", "kind": "integer", "group": "ppo", - "label": "采样步数 n_steps", "default": 128, "min": 1, "max": 100_000, - "help": "每次更新前每个环境采集的步数；近似一个 rollout 的宽度。", - }, - { - "key": "batch_size", "kind": "integer", "group": "ppo", - "label": "批大小 batch_size", "default": 128, "min": 1, "max": 100_000, - "help": "建议 batch_size ≤ n_steps × envs。", - }, - { - "key": "n_epochs", "kind": "integer", "group": "ppo", - "label": "更新轮数 n_epochs", "default": 3, "min": 1, "max": 100, - "help": "每批数据的 SGD 轮数；过大易使 KL 飙升。", - }, - # ---------------------------------------------------------- 运行环境 - { - "key": "envs", "kind": "integer", "group": "runtime", - "label": "并行环境数", "default": 1, "min": 1, "max": 32, - "help": "SubprocVecEnv 下每个 worker 一个进程与 Monitor 文件。", - }, - { - "key": "vec_env", "kind": "select", "group": "runtime", - "label": "向量化方式", "default": "dummy", "choices": [ - {"value": "dummy", "label": "dummy（单进程，调试友好）"}, - {"value": "subproc", "label": "subproc（多进程，吞吐优先）"}, - ], - "help": "envs>1 且追求吞吐时选 subproc。", - }, - { - "key": "device", "kind": "select", "group": "runtime", - "label": "计算设备", "default": "auto", "choices": [ - {"value": "auto", "label": "auto（自动选择）"}, - {"value": "cpu", "label": "cpu"}, - {"value": "cuda", "label": "cuda（NVIDIA）"}, - {"value": "mps", "label": "mps（Apple 芯片）"}, - ], - "help": "CPU 路径最稳；GPU 需与 torch 构建匹配。", - }, - { - "key": "local_selector", "kind": "select", "group": "runtime", - "label": "低层执行器", "default": "geometry", "choices": [ - {"value": "geometry", "label": "geometry（几何选点，默认）"}, - {"value": "rl", "label": "rl（低层 PPO，接口预留）"}, - ], - "help": "rl 仅记录元数据，当前 HTTP 执行路径仍由高层宏动作终点决定。", - }, - # ---------------------------------------------------------- 训练规模 - { - "key": "timesteps", "kind": "integer", "group": "scale", - "label": "总步数 timesteps", "default": 2048, "min": 1, - "max": 10_000_000, "help": "本次训练的环境步总数（续训时为增量步数）。", - }, - { - "key": "max_wall_s", "kind": "number", "group": "scale", - "label": "墙钟上限 (s)", "default": 1200.0, "min": 0.1, "max": 7200.0, - "step": 30, "help": "硬上限：到点由回调主动停止并保存模型。", - }, - { - "key": "eval_episodes", "kind": "integer", "group": "scale", - "label": "评估回合数", "default": 2, "min": 0, "max": 1000, - "help": "训练结束后确定性评估；0 表示跳过。", - }, - { - "key": "verbose", "kind": "select", "group": "scale", - "label": "SB3 日志级别", "default": 1, "choices": [ - {"value": 0, "label": "0 · 静默（只留 JSON 进度）"}, - {"value": 1, "label": "1 · 表格（默认）"}, - {"value": 2, "label": "2 · 调试"}, - ], - "help": "前端已解析 SB3 表格，默认 1 便于对照原始输出。", - }, - # ---------------------------------------------------------- 模型路径 - { - "key": "model", "kind": "text", "group": "paths", - "label": "模型输出路径", "default": "output/hrl/ui/q3.zip", - "help": "相对仓库根；同目录会写入 .json 元数据与 Monitor CSV。", - }, - { - "key": "init", "kind": "text", "group": "paths", "optional": True, - "label": "初始权重 --init", "default": None, - "help": "留空=从零；填上一阶段 .zip 即课程迁移。PPO 超参与步数计数沿用" - "该模型（timesteps 按增量理解，进度条按增量显示）。", - }, - { - "key": "local_model", "kind": "text", "group": "paths", "optional": True, - "label": "低层模型 --local-model", "default": None, - "advanced": True, - "help": "仅 local_selector=rl 时有意义。", - }, - { - "key": "resume", "kind": "bool", "group": "paths", - "label": "续训 --resume", "default": False, - "help": "从模型输出路径已有 .zip 继续，timesteps 按增量理解。", - }, - { - "key": "force", "kind": "bool", "group": "paths", - "label": "覆盖已有模型 --force", "default": False, - "help": "输出路径已存在时必须勾选，否则训练脚本会拒绝覆盖。与续训互斥。", - }, - # ---------------------------------------------------------- 恢复与落盘 - { - "key": "recovery_every", "kind": "integer", "group": "recovery", - "label": "滚动恢复点间隔 --recovery-every", "default": 2000, - "help": "每 N 个环境步原子覆盖写 /_latest.zip，" - "训练开始前先写起步基线；急停（强杀）后最多丢 N 步。0=关闭。", - }, - { - "key": "checkpoint_every", "kind": "integer", "group": "recovery", - "label": "历史 checkpoint 间隔 --checkpoint-every", "default": 20000, - "help": "每 N 个环境步另留一份 _<步数>.zip 供返祖对比；" - "0=只保留滚动恢复点。", - }, - { - "key": "checkpoint_keep", "kind": "integer", "group": "recovery", - "label": "历史 checkpoint 保留数 --checkpoint-keep", "default": 3, - "help": "只保留最近 K 个历史 checkpoint；滚动恢复点与起步基线不受影响。", - }, -] - -CONFIG_GROUPS: list[dict[str, str]] = [ - {"key": "env", "label": "环境与课程", "hint": "题目、课程阶段与奖励口径"}, - {"key": "ppo", "label": "PPO 超参数", "hint": "采样宽度与更新强度"}, - {"key": "runtime", "label": "运行环境", "hint": "并行、设备与低层执行器"}, - {"key": "scale", "label": "训练规模", "hint": "步数、墙钟上限与评估"}, - {"key": "recovery", "label": "恢复与落盘", "hint": "急停恢复点与历史 checkpoint"}, - {"key": "paths", "label": "模型与迁移", "hint": "产物路径、续训与课程迁移"}, -] - -# 预设 = 只覆盖差异字段；后端 from_payload 会补齐其余默认值。 -PRESETS: list[dict[str, Any]] = [ - { - "key": "smoke", - "label": "冒烟（约 1 分钟）", - "note": "64 步、CPU、无评估——只验证链路与界面。", - "payload": { - "problem": 3, "stage": 1, "timesteps": 64, "seed": 1, - "envs": 1, "vec_env": "dummy", "device": "cpu", - "config_profile": "coarse-pretrain", "reward_profile": "time", - "model": "output/hrl/ui/smoke/q3.zip", "eval_episodes": 0, - "rollout_steps": 32, "batch_size": 32, "n_epochs": 2, - "max_wall_s": 180.0, "verbose": 1, "force": True, - }, - }, - { - "key": "local-check", - "label": "本地验证（约 10 分钟）", - "note": "stage 1 单源课程 2048 步 + 2 回合评估。", - "payload": { - "problem": 3, "stage": 1, "timesteps": 2048, "seed": 320001, - "envs": 1, "vec_env": "dummy", "device": "auto", - "config_profile": "production", "reward_profile": "time", - "model": "output/hrl/ui/local/q3.zip", "eval_episodes": 2, - "rollout_steps": 128, "batch_size": 128, "n_epochs": 3, - "max_wall_s": 900.0, "verbose": 1, "force": True, - }, - }, - { - "key": "q3-main", - "label": "Q3 主训练", - "note": "stage 3 八源课程，4 进程并行；建议在训练机上跑。", - "payload": { - "problem": 3, "stage": 3, "timesteps": 200000, "seed": 320001, - "envs": 4, "vec_env": "subproc", "device": "auto", - "config_profile": "production", "reward_profile": "time", - "model": "output/hrl/ui/q3-main/q3.zip", "eval_episodes": 5, - "rollout_steps": 256, "batch_size": 256, "n_epochs": 10, - "max_wall_s": 7200.0, "verbose": 1, - }, - }, - { - "key": "q4-transfer", - "label": "Q4 课程迁移", - "note": "从 Q3 权重迁移到问题 4 末段课程（含定向源）。", - "payload": { - "problem": 4, "stage": 4, "timesteps": 120000, "seed": 420001, - "envs": 4, "vec_env": "subproc", "device": "auto", - "config_profile": "production", "reward_profile": "time", - "model": "output/hrl/ui/q4-main/q4.zip", - "init": "output/hrl/ui/q3-main/q3.zip", "eval_episodes": 5, - "rollout_steps": 256, "batch_size": 256, "n_epochs": 10, - "max_wall_s": 7200.0, "verbose": 1, - }, - }, -] - -# 指标字典：键来自训练脚本 --progress-json 的单行 JSON（短名）与 SB3 表格 -# （归一化去前缀）。前端按此顺序绘图，收敛判据在 server.summarize_convergence。 -METRIC_CATALOG: list[dict[str, Any]] = [ - { - "key": "ep_rew_mean", "label": "平均回合奖励", "role": "primary", - "color": "#4f8cff", "format": "number", "better": "up", - "help": "最近若干回合的平均回报；主要收敛判据。", - }, - { - "key": "ep_len_mean", "label": "平均回合长度", "role": "secondary", - "color": "#38bdf8", "format": "number", "better": "neutral", - "help": "回合步数；与奖励同时上升通常代表策略更完整而非更快。", - }, - { - "key": "approx_kl", "label": "近似 KL", "role": "stability", - "color": "#f59e0b", "format": "sci", "better": "down", - "help": "策略更新幅度；> 0.05 视为更新过猛。", - }, - { - "key": "explained_variance", "label": "解释方差", "role": "stability", - "color": "#34d399", "format": "number", "better": "up", - "help": "价值函数拟合优度；< 0 表示还不如均值预测。", - }, - { - "key": "value_loss", "label": "价值损失", "role": "stability", - "color": "#f87171", "format": "sci", "better": "down", - "help": "critic 拟合误差。", - }, - { - "key": "policy_gradient_loss", "label": "策略梯度损失", "role": "stability", - "color": "#c084fc", "format": "sci", "better": "neutral", - "help": "actor 的代理损失；量级小且符号可变属正常。", - }, - { - "key": "entropy_loss", "label": "策略熵", "role": "exploration", - "color": "#fbbf24", "format": "number", "better": "neutral", - "help": "探索度；若长期贴近 0 说明策略过早确定。", - }, - { - "key": "n_updates", "label": "更新次数", "role": "throughput", - "color": "#93a1bd", "format": "integer", "better": "up", - "help": "已完成的 PPO 更新轮数。", - }, - { - "key": "fps", "label": "吞吐 FPS", "role": "throughput", - "color": "#22d3ee", "format": "number", "better": "up", - "help": "每秒环境步数，反映硬件与并行配置是否吃满。", - }, -] - -_METRIC_BY_KEY = {item["key"]: item for item in METRIC_CATALOG} - - -def metric_keys() -> list[str]: - return [item["key"] for item in METRIC_CATALOG] - - -def metric_meta(key: str) -> dict[str, Any]: - return _METRIC_BY_KEY.get(key, {"key": key, "label": key, "role": "other", - "color": "#93a1bd", "format": "number", - "better": "neutral", "help": ""}) - - -def config_schema() -> dict[str, Any]: - """返回给前端的完整配置契约（字段 + 分组 + 预设 + 指标字典）。""" - return { - "fields": CONFIG_FIELDS, - "groups": CONFIG_GROUPS, - "presets": [{"key": item["key"], "label": item["label"], - "note": item["note"], "payload": dict(item["payload"])} - for item in PRESETS], - "metrics": METRIC_CATALOG, - } - - -def preset_payloads() -> dict[str, dict[str, Any]]: - return {item["key"]: dict(item["payload"]) for item in PRESETS} diff --git a/src/mcm26b/training_ui/server.py b/src/mcm26b/training_ui/server.py deleted file mode 100644 index eb956d2..0000000 --- a/src/mcm26b/training_ui/server.py +++ /dev/null @@ -1,977 +0,0 @@ -"""训练前端服务器：在评测前端同一套「本地回环 HTTP + 静态白名单」模式上， -额外托管一个受控的训练子进程，并把 stdout/JSON 进度流做成轮询 API。 - -设计边界： -- 只监听 127.0.0.1，只允许白名单静态文件； -- 只允许一个训练任务，命令由服务端用参数列表构造，不经过 shell； -- 支持启动/暂停/继续/停止/状态查询；暂停与停止通过**控制文件**下发， - POSIX 与 Windows 行为一致（详见 :mod:`mcm26b.process_control`）； -- 训练进程通过 ``--progress-json`` 输出单行 JSON 进度，服务端同时兼容 - Stable-Baselines3 的表格输出和 Monitor CSV。 - -界面契约（供 ``static/`` 使用）： -- ``GET /api/train/schema`` 字段/分组/预设/指标字典（``schema.py`` 单一出处） -- ``GET /api/train/state`` 快照；``after_metric=N`` 只回增量指标 -- ``POST /api/train/start`` 以部分字段启动（其余取默认值） -- ``POST /api/train/pause`` 请求训练进程在 step 边界挂起 -- ``POST /api/train/resume`` 解除挂起 -- ``POST /api/train/stop`` 请求优雅停止，超时后强杀进程树 -""" - -from __future__ import annotations - -import dataclasses -import json -import os -import re -import subprocess -import sys -import threading -import time -from collections import deque -from dataclasses import dataclass -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path -from typing import Any -from urllib.parse import parse_qs, urlsplit - -from ..process_control import (DEFAULT_GRACE_S, spawn_kwargs, - terminate_process_tree, write_control, - write_json_atomic) -from .analysis import (MAX_LOG_LINES, MAX_METRICS, describe_process, finite_float, - process_pid, summarize_convergence) -from .demo import DEMO_SCENARIOS, demo_snapshot -from .schema import config_schema - -__all__ = ["TrainingJob", "TrainingUI", "run_training_ui", "build_command", - "summarize_convergence", "describe_process", "discover_recovery"] - -STATIC_DIR = Path(__file__).resolve().parent / "static" -REPO_ROOT = Path(__file__).resolve().parents[3] -TRAIN_SCRIPT = REPO_ROOT / "scripts" / "train_hrl_high.py" -# 控制文件集中放在 output/ 下（已 gitignore），job 之间互不干扰 -CONTROL_DIR = REPO_ROOT / "output" / "hrl" / "ui" / "control" -# 从发出停止请求到强杀之间留给训练脚本收尾的秒数 -STOP_GRACE_S = DEFAULT_GRACE_S -STATIC_FILES = { - "/": ("index.html", "text/html; charset=utf-8"), - "/index.html": ("index.html", "text/html; charset=utf-8"), - "/app.js": ("app.js", "application/javascript; charset=utf-8"), - "/charts.js": ("charts.js", "application/javascript; charset=utf-8"), - "/style.css": ("style.css", "text/css; charset=utf-8"), -} - -_SB3_SECTIONS = ("rollout/", "time/", "train/", "system/") -# 路径/字符串参数只拒绝控制字符（含 \r\n\t 与 DEL），不做 shell 元字符过滤： -# 命令走参数列表、不经过 shell，这里只是防日志与命令行被注入控制序列。 -# 注意必须用单反斜杠转义——写成 r"...\\x00" 会变成"字面反斜杠到反斜杠"的 -# 字符类，从而把所有含 `\` 或看起来像路径分隔的输入误判为非法。 -_SAFE_ARG_RE = re.compile(r"^[^\x00-\x1f\x7f]{1,512}$") - - -def _normalize_metric(key: str) -> str: - """把 SB3 的 ``rollout/ep_rew_mean`` 与进度 JSON 的 ``ep_rew_mean`` - 归一化到同一短名，便于前端绘制。""" - key = key.strip().replace(" ", "_") - if "/" in key: - key = key.rsplit("/", 1)[-1] - return key - - -@dataclass(frozen=True) -class TrainConfig: - problem: int = 3 - stage: int = 1 - timesteps: int = 2048 - seed: int = 320001 - envs: int = 1 - vec_env: str = "dummy" - device: str = "auto" - config_profile: str = "production" - reward_profile: str = "time" - model: str = "output/hrl/ui/q3.zip" - init: str | None = None - resume: bool = False - force: bool = False - eval_episodes: int = 2 - rollout_steps: int = 128 - batch_size: int = 128 - n_epochs: int = 3 - max_wall_s: float = 1200.0 - local_selector: str = "geometry" - local_model: str | None = None - # 落盘/恢复：默认值与 scripts/train_hrl_high.py 保持一致 - checkpoint_every: int = 20000 - checkpoint_keep: int = 3 - recovery_every: int = 2000 - verbose: int = 1 - - @classmethod - def from_payload(cls, payload: dict[str, Any]) -> "TrainConfig": - if not isinstance(payload, dict): - raise ValueError("请求体必须是 JSON 对象") - defaults = cls() - data: dict[str, Any] = {} - # 用 dataclasses.fields 而不是 cls.__dataclass_fields__：后者是内部 - # 实现细节，pylint 看不到它，也无法保证未来版本仍存在。 - for field in dataclasses.fields(cls): - if field.name in payload and payload[field.name] is not None: - data[field.name] = payload[field.name] - cfg = cls(**{**defaults.__dict__, **data}) - cfg.validate() - return cfg - - def validate(self) -> None: - if self.problem not in (3, 4): - raise ValueError("problem 必须是 3 或 4") - if self.stage not in (1, 2, 3, 4): - raise ValueError("stage 必须在 1..4") - if not 1 <= int(self.timesteps) <= 10_000_000: - raise ValueError("timesteps 必须在 1..10,000,000") - if bool(self.resume) and bool(self.force): - raise ValueError("--resume 与 --force 互斥：续训不可能同时覆盖") - if not 1 <= int(self.envs) <= 32: - raise ValueError("envs 必须在 1..32") - if self.vec_env not in ("dummy", "subproc"): - raise ValueError("vec_env 必须是 dummy 或 subproc") - if self.device not in ("auto", "cpu", "cuda", "mps"): - raise ValueError("device 必须是 auto/cpu/cuda/mps") - if self.config_profile not in ("production", "coarse-pretrain"): - raise ValueError("config_profile 非法") - if self.reward_profile not in ("time", "prompt"): - raise ValueError("reward_profile 非法") - if self.local_selector not in ("geometry", "rl"): - raise ValueError("local_selector 非法") - if not 1 <= int(self.rollout_steps) <= int(self.timesteps): - raise ValueError("rollout_steps 必须在 1..timesteps") - if not 1 <= int(self.n_epochs) <= 100: - raise ValueError("n_epochs 必须在 1..100") - if not 0.0 < float(self.max_wall_s) <= 7200.0: - raise ValueError("max_wall_s 必须在 0..7200 秒") - if int(self.eval_episodes) < 0: - raise ValueError("eval_episodes 不能为负") - if not 0 <= int(self.checkpoint_every) <= 10_000_000: - raise ValueError("checkpoint_every 必须在 0..10,000,000") - if not 1 <= int(self.checkpoint_keep) <= 100: - raise ValueError("checkpoint_keep 必须在 1..100") - if not 0 <= int(self.recovery_every) <= 10_000_000: - raise ValueError("recovery_every 必须在 0..10,000,000") - for label, value in (("model", self.model), ("init", self.init), - ("local_model", self.local_model)): - if value is None: - continue - if not isinstance(value, str) or not _SAFE_ARG_RE.match(value): - raise ValueError(f"{label} 路径含非法字符") - - @property - def model_path(self) -> Path: - return (REPO_ROOT / self.model).resolve() if not Path(self.model).is_absolute() \ - else Path(self.model).resolve() - - def command(self) -> list[str]: - return build_command(self) - - -def _int_query(params: dict[str, list[str]], name: str, default: int | None, - low: int, high: int, *, optional: bool = False) -> int | None: - """解析整数查询参数；越界或非整数一律报错，不用静默钳制掩盖调用错误。""" - if name not in params: - return None if optional else default - raw = (params[name] or [""])[0].strip() - if raw == "": - return None if optional else default - try: - value = int(raw) - except ValueError: - raise ValueError(f"{name} 必须是整数") from None - if not low <= value <= high: - raise ValueError(f"{name} 必须在 {low}..{high}") - return value - - -def build_command(config: TrainConfig) -> list[str]: - if not TRAIN_SCRIPT.is_file(): - raise FileNotFoundError(f"训练脚本不存在: {TRAIN_SCRIPT}") - cmd = [ - sys.executable, str(TRAIN_SCRIPT), - "--problem", str(config.problem), - "--stage", str(config.stage), - "--timesteps", str(config.timesteps), - "--seed", str(config.seed), - "--envs", str(config.envs), - "--vec-env", config.vec_env, - "--device", config.device, - "--config-profile", config.config_profile, - "--reward-profile", config.reward_profile, - "--model", str(config.model_path), - "--eval-episodes", str(config.eval_episodes), - "--rollout-steps", str(config.rollout_steps), - "--batch-size", str(config.batch_size), - "--n-epochs", str(config.n_epochs), - "--max-wall-s", str(config.max_wall_s), - "--local-selector", config.local_selector, - "--checkpoint-every", str(config.checkpoint_every), - "--checkpoint-keep", str(config.checkpoint_keep), - "--recovery-every", str(config.recovery_every), - "--progress-json", - "--verbose", str(config.verbose), - ] - if config.resume: - cmd.append("--resume") - if config.force: - cmd.append("--force") - if config.init: - cmd.extend(["--init", str((REPO_ROOT / config.init).resolve() - if not Path(config.init).is_absolute() - else config.init)]) - if config.local_model: - cmd.extend(["--local-model", str(config.local_model)]) - return cmd - - -def discover_recovery(run_dir: Path, model_path: Path) -> dict[str, Any]: - """找出"现在就能继续训练"的那份产物。 - - 优先级： - 1. ``/resume.json``——训练脚本写的权威指针（校验目标文件真实存在）； - 2. 否则按 mtime 取最新：最终模型 / 滚动恢复点 / 历史 checkpoint / 异常兜底。 - - 只认**已完整落盘**的文件：模型的 zip 与 json 都是原子替换写的，所以不会 - 出现"文件在、但打不开"的半截快照。 - """ - run_dir = Path(run_dir) - model_path = Path(model_path) - hint_file = run_dir / "resume.json" - hint = _read_json_file(hint_file) - - candidates: list[tuple[float, str, str]] = [] - - def _consider(path: Path, reason: str) -> None: - try: - stat = path.stat() - except OSError: - return - if stat.st_size > 0: - candidates.append((stat.st_mtime, str(path), reason)) - - _consider(model_path, "final_model") - ckpt_dir = run_dir / "checkpoints" - if ckpt_dir.is_dir(): - _consider(ckpt_dir / f"{model_path.stem}_ckpt_latest.zip", "recovery") - for path in ckpt_dir.glob(f"{model_path.stem}_ckpt_[0-9]*.zip"): - if path.stem.rsplit("_", 1)[-1].isdigit(): - _consider(path, "checkpoint") - _consider(run_dir / f"{model_path.stem}_emergency.zip", "emergency") - - state: dict[str, Any] = { - "can_resume": False, "model": None, "timesteps": None, - "reason": "none", "updated_unix_s": None, "source": "none", - "hint_file": str(hint_file), "candidates": len(candidates), - } - if hint and hint.get("model") and Path(str(hint["model"])).is_file(): - state.update({ - "model": str(hint["model"]), - "timesteps": _int_or_none(hint.get("timesteps")), - "reason": str(hint.get("reason") or "resume_hint"), - "updated_unix_s": _float_or_none(hint.get("created_unix_s")), - "source": "resume.json", - }) - elif candidates: - mtime, path, reason = max(candidates, key=lambda item: item[0]) - state.update({ - "model": path, "reason": reason, - "updated_unix_s": mtime, "source": "scan", - }) - if state["model"]: - state["can_resume"] = Path(str(state["model"])).is_file() - return state - - -def _read_json_file(path: Path) -> dict[str, Any] | None: - """读 JSON 文件；缺失或损坏返回 None（恢复指针不是关键路径，不能因此崩）。""" - try: - payload = json.loads(Path(path).read_text(encoding="utf-8")) - except (OSError, ValueError, UnicodeDecodeError): - return None - return payload if isinstance(payload, dict) else None - - -def _int_or_none(value: Any) -> int | None: - try: - return int(value) - except (TypeError, ValueError): - return None - - -def _float_or_none(value: Any) -> float | None: - try: - return float(value) - except (TypeError, ValueError): - return None - - -class TrainingJob: - """单个训练子进程的生命周期、日志和指标缓存。 - - ``control_dir`` 是控制文件的落地目录；默认与模型同目录，服务端则统一放到 - ``output/hrl/ui/control``（已被 gitignore）。 - """ - - def __init__(self, config: TrainConfig, control_dir: Path | None = None): - self.config = config - self.job_id = f"job-{int(time.time() * 1000)}" - self.control_path = (Path(control_dir) if control_dir is not None - else config.model_path.parent) / \ - f"{self.job_id}.control.json" - # 控制文件是暂停/停止的唯一可靠通道：Windows 上没有 SIGSTOP， - # 且 os.kill(SIGTERM) 会退化成硬杀，信号无法承载"优雅收尾"语义。 - self.command = config.command() + ["--control-file", str(self.control_path)] - self.created_at = time.time() - self.started_at: float | None = None - self.finished_at: float | None = None - self.returncode: int | None = None - # created/running/paused/stopping/stopped/exited/failed - self.status = "created" - self.error: str | None = None - self.forced_kill = False # 停止请求超时后是否动用了强杀 - self.process: subprocess.Popen | None = None - self.logs: deque[str] = deque(maxlen=MAX_LOG_LINES) - self.metrics: deque[dict[str, Any]] = deque(maxlen=MAX_METRICS) - self.monitor: list[dict[str, Any]] = [] - self._lock = threading.RLock() - self._reader: threading.Thread | None = None - self._stopper: threading.Thread | None = None - self._table_sections: list[tuple[str, dict[str, Any]]] = [] - self._table_steps: float | None = None # 当前表的 total_timesteps - self._pending_table: dict[str, Any] | None = None # 未见刻度的半张表 - self._last_table_steps: float | None = None # 上一张表已知刻度（截断兜底） - self._last_json_steps: float | None = None # 最近一条 progress JSON 的步数 - self._last_monitor_sig: tuple[int, int] | None = None - - # ------------------------------------------------------------ lifecycle - def start(self) -> None: - with self._lock: - if self.process is not None and self.process.poll() is None: - raise RuntimeError("已有训练任务在运行") - # 先落一份"不暂停、不停止"的初始控制文件，训练脚本首步就能读到； - # 否则脚本要等到第一次轮询才区分"文件缺失"与"无指令"。 - write_control(self.control_path, paused=False, stop=False) - env = os.environ.copy() - env["PYTHONUNBUFFERED"] = "1" - env["HRL_UI_JOB_ID"] = self.job_id - self.process = subprocess.Popen( # pylint: disable=consider-using-with - self.command, - cwd=str(REPO_ROOT), - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - bufsize=1, - env=env, - **spawn_kwargs(), - ) - self.started_at = time.time() - self.status = "running" - self._reader = threading.Thread( - target=self._read_loop, name=f"{self.job_id}-reader", daemon=True) - self._reader.start() - - def pause(self) -> bool: - """请求训练进程在下一个 step 边界挂起（跨平台：控制文件）。""" - with self._lock: - process = self.process - if process is None or process.poll() is not None: - return False - if self.status not in ("running", "paused"): - return False - write_control(self.control_path, paused=True, stop=False) - self.status = "paused" - return True - - def resume(self) -> bool: - """解除挂起；训练从暂停的那个 step 继续。""" - with self._lock: - process = self.process - if process is None or process.poll() is not None: - return False - if self.status not in ("paused", "running"): - return False - write_control(self.control_path, paused=False, stop=False) - self.status = "running" - return True - - def stop(self) -> bool: - """请求优雅停止；超时后由看门狗强杀整棵进程树。 - - 控制文件先于信号：Windows 上 SIGTERM 等价于 TerminateProcess，训练 - 脚本"保存模型 + 写元数据"的收尾分支不会执行；控制文件则两个平台都能 - 命中同一段协作式收尾。强杀放到独立线程，避免 HTTP 请求被 grace 等待卡住。 - """ - with self._lock: - process = self.process - if process is None or process.poll() is not None: - return False - write_control(self.control_path, paused=False, stop=True) - self.status = "stopping" - if self._stopper is None or not self._stopper.is_alive(): - self._stopper = threading.Thread( - target=self._stop_watchdog, args=(process,), - name=f"{self.job_id}-stopper", daemon=True) - self._stopper.start() - return True - - def _stop_watchdog(self, process: subprocess.Popen) -> None: - """等 grace 让协作式停止生效，超时强杀并记下恢复点。""" - forced = terminate_process_tree(process, grace_s=STOP_GRACE_S) - with self._lock: - self.forced_kill = forced - if forced and not self.error: - self.error = (f"训练进程未在 {STOP_GRACE_S:.0f}s 内响应停止请求，" - f"已强制结束进程树") - if forced: - self._write_forced_stop(process.pid) - - def _write_forced_stop(self, pid: int) -> None: - """强杀后落一份 `<模型目录>/forced_stop.json`，写明"从哪继续"。 - - 强杀没有给训练脚本任何收尾机会，所以这里由前端补上审计与恢复指引： - 磁盘上仍有原子落盘的滚动恢复点 / 历史 checkpoint，``recovery.model`` - 就是可直接 ``--resume`` 的目标。 - """ - run_dir = self.config.model_path.parent - try: - recovery = discover_recovery(run_dir, self.config.model_path) - write_json_atomic(run_dir / "forced_stop.json", { - "event": "forced_stop", - "job_id": self.job_id, - "pid": int(pid), - "created_unix_s": time.time(), - "reason": "grace_timeout", - "grace_s": STOP_GRACE_S, - "recovery": recovery, - "note": ("训练进程未响应控制文件，已强杀整棵进程树；" - "把 recovery.model 作为 --resume 目标即可继续" - "（timesteps 为追加步数）"), - }) - except OSError as exc: # 标记文件写不了也不能影响停止流程 - with self._lock: - if not self.error: - self.error = f"强杀后写恢复标记失败：{exc}" - - def wait_for_exit(self, timeout: float | None = None) -> bool: - """等待停止流程收尾（优雅退出或强杀）；供前端进程退出前调用。 - - 训练子进程在独立进程组里，前端先退出并不会带走它。若不在这里等， - 前端一退出，子进程会成为孤儿——虽然控制文件仍会让它自行收尾，但 - 日志与退出码就没人接了。 - """ - thread = self._stopper - if thread is not None: - thread.join(timeout) - process = self.process - return process is None or process.poll() is not None - - def _read_loop(self) -> None: - process = self.process - assert process is not None and process.stdout is not None - try: - for raw in process.stdout: - line = raw.rstrip("\n") - self._on_line(line) - process.wait() - except Exception as exc: # pragma: no cover - 子进程读取异常 - with self._lock: - self.error = f"{type(exc).__name__}: {exc}" - finally: - with self._lock: - self.finished_at = time.time() - if self.returncode is None: - self.returncode = process.poll() - if self.status == "stopping": - self.status = "stopped" - elif self.returncode == 0: - self.status = "exited" - else: - self.status = "failed" - self._flush_table() - # 最后半张表（Ctrl+C / 崩溃时没打印 total_timesteps）按末尾 - # 已知刻度兜底提交，宁可标注来源也不丢最后一段曲线 - self._flush_pending_with_fallback() - - def _on_line(self, line: str) -> None: - with self._lock: - self.logs.append(line) - self._sync_control_events(line) - self._parse_progress_json(line) - self._parse_sb3_table(line) - self._refresh_monitor() - - def _sync_control_events(self, line: str) -> None: - """用训练脚本的回显校正前端状态。 - - 正常情况下状态由前端自己设置；但外部直接改控制文件、或脚本比前端更早 - 感知到指令时，以脚本回显的 ``paused`` / ``resumed`` / ``stop_requested`` - 为准更真实。 - """ - stripped = line.strip() - if not stripped.startswith("{") or '"event"' not in stripped: - return - try: - payload = json.loads(stripped) - except ValueError: - return - if not isinstance(payload, dict): - return - event = payload.get("event") - if event not in ("paused", "resumed", "stop_requested"): - return - with self._lock: - if event == "paused" and self.status == "running": - self.status = "paused" - elif event == "resumed" and self.status == "paused": - self.status = "running" - elif event == "stop_requested" and self.status in ("running", "paused"): - self.status = "stopping" - - # ------------------------------------------------------------ parsers - def _parse_progress_json(self, line: str) -> None: - stripped = line.strip() - if not stripped.startswith("{") or "progress" not in stripped: - return - try: - payload = json.loads(stripped) - except ValueError: - return - if not isinstance(payload, dict) or payload.get("event") != "progress": - return - point = {_normalize_metric(str(key)): value - for key, value in payload.items() if key != "event"} - steps = finite_float(point.get("timesteps")) - if steps is None: - steps = finite_float(point.get("total_timesteps")) - if steps is not None: - with self._lock: - self._last_json_steps = steps - self._append_metric(point) - - def _parse_sb3_table(self, line: str) -> None: - """兼容 SB3 verbose=1 的 ``| key | value |`` 表格。 - - SB3 每张表按 rollout/ → time/ → train/ → system/ 分段，``total_timesteps`` - 出现在 time/ 段——也就是要等同一张表的 rollout/ 行先读进来之后才拿到 - 横轴。因此不能在段头立刻提交：那会把还没有刻度的 rollout/ 行丢掉，或 - 把它们算到上一张表的刻度上。 - - 规则： - - 每次"回到 rollout/ 段"视为新一张表开始，先把上一张已封闭的表提交； - - 表格分隔线表示本表结束，此时才允许用上一张表的刻度兜底（截断表）； - - 从未封闭过的半张表一直挂到进程结束才兜底提交，避免用旧刻度冒充。 - """ - if not line.startswith("|"): - if line.startswith("-"): - self._flush_table() - return - cells = [cell.strip() for cell in line.split("|")] - if len(cells) < 4: - return - raw_key, raw_value = cells[1], cells[2] - if raw_key in _SB3_SECTIONS: - if cells[0] != "": - return - with self._lock: - if raw_key == _SB3_SECTIONS[0] and self._table_sections: - self._flush_table() - if not self._table_sections or self._table_sections[-1][0] != raw_key: - self._table_sections.append((raw_key, {})) - return - key = _normalize_metric(raw_key) - value = finite_float(raw_value) - if not key or value is None: - return - with self._lock: - if not self._table_sections: - self._table_sections.append((_SB3_SECTIONS[0], {})) - self._table_sections[-1][1][key] = value - if key == "total_timesteps": - self._table_steps = value - - def _flush_table(self) -> None: - """提交缓存中的 SB3 表。 - - 刻度优先级：本表的 ``total_timesteps`` > 上一张表的已知刻度（仅在遇到 - 分隔线即"确认整表结束"时才允许兜底，见 ``_on_line``）。完全没有刻度 - 就丢弃——伪造横轴会在曲线上画出重复的幽灵点。与 JSON 进度行同一步数时 - 跳过表格版本（JSON 字段更全），表格只用于补 clip_fraction/learning_rate。 - """ - with self._lock: - sections = self._table_sections - steps = self._table_steps - self._table_sections = [] - self._table_steps = None - last_json = self._last_json_steps - if not sections: - return - table: dict[str, Any] = {} - for _name, items in sections: - table.update(items) - if steps is None: - steps = finite_float(table.get("total_timesteps")) - if steps is None: - # 半张表：交给调用方决定是否已确认结束（终止时按末尾兜底提交） - with self._lock: - self._pending_table = table - return - with self._lock: - self._last_table_steps = steps - if last_json is not None: - tolerance = max(1.0, 0.25 * steps / max(1.0, self.config.rollout_steps)) - if abs(steps - last_json) <= tolerance: - return - if "timesteps" not in table: - table["timesteps"] = steps - self._append_metric(table) - - def _flush_pending_with_fallback(self) -> None: - """进程结束时提交最后半张表：用上一次已知刻度兜底，避免丢数据。""" - with self._lock: - pending = self._pending_table - self._pending_table = None - steps = self._last_table_steps - if not pending or steps is None: - return - if "timesteps" not in pending: - pending = {**pending, "timesteps": steps} - self._append_metric(pending) - - def _append_metric(self, point: dict[str, Any]) -> None: - clean: dict[str, Any] = {} - for key, value in point.items(): - number = finite_float(value) - clean[key] = number if number is not None else value - # 补齐 timesteps：progress JSON 优先，SB3 表用 total_timesteps。 - if "timesteps" not in clean and "total_timesteps" in clean: - clean["timesteps"] = clean["total_timesteps"] - with self._lock: - if self.metrics and clean == self.metrics[-1]: - return - self.metrics.append(clean) - - def _refresh_monitor(self) -> None: - """训练脚本写入的 Monitor CSV：作为逐 episode 奖励的备用来源。""" - parent = self.config.model_path.parent - files = sorted(parent.glob("high_curve_*_env*.monitor.csv")) \ - if parent.is_dir() else [] - if not files: - return - signature = (len(files), max(path.stat().st_mtime_ns for path in files)) - if signature == self._last_monitor_sig: - return - rows: list[dict[str, Any]] = [] - for path in files: - try: - lines = path.read_text(encoding="utf-8", errors="replace").splitlines() - except OSError: - continue - header: list[str] | None = None - for line in lines: - if line.startswith("#") or not line.strip(): - continue - if header is None: - header = [item.strip() for item in line.split(",")] - continue - values = line.split(",") - if len(values) < len(header): - continue - row = {} - valid = True - for key, raw in zip(header, values): - number = finite_float(raw) - if number is None: - valid = False - break - row[key] = number - if valid: - rows.append(row) - with self._lock: - self.monitor = rows[-MAX_METRICS:] - self._last_monitor_sig = signature - - # ------------------------------------------------------------ snapshot - def snapshot(self, *, log_tail: int = 400, metric_limit: int = 2000, - after_metric: int | None = None) -> dict[str, Any]: - with self._lock: - now = self.finished_at or time.time() - elapsed = (now - self.started_at) if self.started_at else 0.0 - logs = list(self.logs)[-max(1, int(log_tail)):] - total = len(self.metrics) - if after_metric is None: - metrics = list(self.metrics)[-max(1, int(metric_limit)):] - offset = total - len(metrics) - else: - start = max(0, min(int(after_metric), total)) - metrics = list(self.metrics)[start:] - offset = start - # metric_end 是本批点在序列中的绝对末端下标：服务端 deque 满 5000 - # 后会丢弃最老的点，客户端只有拿到这个值才能区分"续接"与"服务端 - # 已滑动窗口"，否则同一批点会被重复追加。 - metric_end = offset + len(metrics) - monitor = self.monitor[-max(1, int(metric_limit)):] - config = dict(self.config.__dict__) - command = list(self.command) - model_path = str(self.config.model_path) - payload = { - "job_id": self.job_id, - "status": self.status, - "pid": process_pid(self.process), - "created_at": self.created_at, - "started_at": self.started_at, - "finished_at": self.finished_at, - "elapsed_s": round(elapsed, 3), - "returncode": self.returncode, - "error": self.error, - "forced_kill": self.forced_kill, - "command": command, - "model": model_path, - "config": config, - "log_lines": logs, - "log_total": len(self.logs), - "metrics": metrics, - "metric_offset": offset, - "metric_end": metric_end, - "metric_total": total, - "monitor": monitor, - } - payload["convergence"] = summarize_convergence(metrics, monitor) - payload["process"] = describe_process(payload) - # 恢复指针在锁外做文件扫描：即使训练被强杀，这里也能给出可续训的产物 - payload["recovery"] = discover_recovery( - self.config.model_path.parent, self.config.model_path) - return payload - - -class TrainingUI: - """训练前端管理器：持有唯一 TrainingJob 与 HTTP 服务器。""" - - def __init__(self, ui_port: int = 8036): - self.ui_port = int(ui_port) - self.job: TrainingJob | None = None - self._lock = threading.RLock() - try: - self.httpd = ThreadingHTTPServer(("127.0.0.1", self.ui_port), _UIHandler) - except OSError as exc: - raise OSError( - f"无法在 127.0.0.1:{self.ui_port} 启动训练前端（{exc}）；" - f"换一个 --ui-port 或先结束占用该端口的进程") from exc - self.httpd.daemon_threads = True - self.httpd.training_ui = self - self._serving = False - - # ------------------------------------------------------------ API - def start(self, payload: dict[str, Any]) -> dict[str, Any]: - config = TrainConfig.from_payload(payload) - with self._lock: - if self.job is not None and self.job.status in ( - "created", "running", "paused", "stopping"): - raise RuntimeError("已有训练任务在运行，请先停止") - self.job = TrainingJob(config, control_dir=CONTROL_DIR) - try: - self.job.start() - except Exception: - self.job.status = "failed" - self.job.error = "训练进程启动失败" - raise - # 与 snapshot() 同形：HTTP 层直接把返回值当响应体，不再二次拆包 - return {"job": self.job.snapshot(log_tail=1)} - - def pause(self) -> bool: - with self._lock: - return bool(self.job and self.job.pause()) - - def resume(self) -> bool: - with self._lock: - return bool(self.job and self.job.resume()) - - def stop(self) -> bool: - with self._lock: - return bool(self.job and self.job.stop()) - - def snapshot(self, *, log_tail: int, metric_limit: int, - after_metric: int | None = None) -> dict[str, Any]: - with self._lock: - if self.job is None: - return {"job": None, "message": "尚未启动训练"} - return {"job": self.job.snapshot(log_tail=log_tail, - metric_limit=metric_limit, - after_metric=after_metric)} - - @property - def is_serving(self) -> bool: - return self._serving - - def serve_forever(self) -> None: - self._serving = True - try: - self.httpd.serve_forever() - finally: - self._serving = False - - def shutdown(self) -> None: - job = self.job - if job is not None and job.stop(): - # 留出收尾落盘的时间，避免前端先退出把训练子进程留成孤儿 - job.wait_for_exit(timeout=STOP_GRACE_S + 5.0) - if self._serving: - self.httpd.shutdown() - self.httpd.server_close() - - -class _UIHandler(BaseHTTPRequestHandler): - protocol_version = "HTTP/1.1" - server_version = "MCM26B-Training-UI/1.0" - - def log_message(self, format, *args): # noqa: A002 # pylint: disable=redefined-builtin - """覆写 stdlib 签名以吞掉访问日志；参数名沿用 ``format`` 不能改。""" - - @property - def ui(self) -> TrainingUI: - return self.server.training_ui # type: ignore[attr-defined] - - def do_GET(self): - parts = urlsplit(self.path) - if parts.path in STATIC_FILES: - self._serve_static(parts.path) - elif parts.path == "/api/train/schema": - self._send_json(200, config_schema()) - elif parts.path == "/api/train/state": - params = parse_qs(parts.query) - if (params.get("preview") or ["0"])[0] in ("1", "true", "yes"): - # 离线预览：返回合成快照，不触碰任何训练进程 - scenario = (params.get("scenario") or ["rising"])[0] - if scenario not in DEMO_SCENARIOS: - self._send_json(400, {"error": f"未知预览场景 {scenario}"}) - return - self._send_json(200, {"job": demo_snapshot(log_tail=400, - scenario=scenario)}) - return - try: - log_tail = _int_query(params, "log_tail", 400, 1, MAX_LOG_LINES) - metric_limit = _int_query(params, "metric_limit", 2000, 1, - MAX_METRICS) - after = _int_query(params, "after_metric", None, 0, - MAX_METRICS * 2, optional=True) - except ValueError as exc: - self._send_json(400, {"error": str(exc)}) - return - self._send_json(200, self.ui.snapshot( - log_tail=log_tail, metric_limit=metric_limit, - after_metric=after)) - else: - self._send_json(404, {"error": f"未知路径 {parts.path}"}) - - def do_POST(self): - parts = urlsplit(self.path) - payload = self._read_json() - if payload is None: - self._send_json(400, {"error": "请求体必须是合法 JSON"}) - return - if not isinstance(payload, dict): - self._send_json(400, {"error": "请求体必须是 JSON 对象"}) - return - try: - if parts.path == "/api/train/start": - self._send_json(200, self.ui.start(payload)) - elif parts.path == "/api/train/pause": - self._send_json(200, {"paused": self.ui.pause()}) - elif parts.path == "/api/train/resume": - self._send_json(200, {"resumed": self.ui.resume()}) - elif parts.path == "/api/train/stop": - self._send_json(200, {"stopped": self.ui.stop()}) - else: - self._send_json(404, {"error": f"未知路径 {parts.path}"}) - except (ValueError, RuntimeError, FileNotFoundError, - PermissionError) as exc: - self._send_json(409, {"error": str(exc)}) - except OSError as exc: - self._send_json(409, {"error": f"操作系统错误：{exc}"}) - except Exception as exc: # pragma: no cover - 兜底，避免连接被静默中断 - self._send_json(500, {"error": f"服务端内部错误：{type(exc).__name__}: {exc}"}) - - def _serve_static(self, path: str) -> None: - filename, content_type = STATIC_FILES[path] - data = (STATIC_DIR / filename).read_bytes() - self.send_response(200) - self.send_header("Content-Type", content_type) - self.send_header("Content-Length", str(len(data))) - self.send_header("Cache-Control", "no-store") - self.end_headers() - self.wfile.write(data) - - def _read_json(self) -> dict[str, Any] | None: - try: - length = int(self.headers.get("Content-Length") or 0) - except ValueError: - return None - if length <= 0 or length > 1_000_000: - return {} - data = self.rfile.read(length) - try: - return json.loads(data.decode("utf-8")) - except (ValueError, UnicodeDecodeError): - return None - - def _send_json(self, status: int, payload: dict[str, Any]) -> None: - data = json.dumps(payload, ensure_ascii=False, allow_nan=False).encode("utf-8") - self.send_response(status) - self.send_header("Content-Type", "application/json; charset=utf-8") - self.send_header("Content-Length", str(len(data))) - self.send_header("Cache-Control", "no-store") - self.end_headers() - self.wfile.write(data) - - -def run_training_ui(argv=None) -> int: - import argparse - - parser = argparse.ArgumentParser(description="HRL 模型训练前端（本地回环）") - parser.add_argument("--ui-port", type=int, default=8036) - parser.add_argument("--no-browser", action="store_true") - parser.add_argument("--print-schema", action="store_true", - help="打印配置契约（字段/分组/预设/指标）后退出") - parser.add_argument("--print-preview", action="store_true", - help="打印离线示例快照 JSON 后退出") - args = parser.parse_args(argv) - if args.print_schema: - print(json.dumps(config_schema(), ensure_ascii=False, indent=2)) - return 0 - if args.print_preview: - print(json.dumps({"job": demo_snapshot(log_tail=200)}, - ensure_ascii=False, indent=2)) - return 0 - try: - ui = TrainingUI(ui_port=args.ui_port) - except OSError as exc: - print(f"[训练前端] {exc}", file=sys.stderr) - return 2 - url = f"http://127.0.0.1:{args.ui_port}/" - # flush=True：界面常用 `> log 2>&1 &` 方式后台启动，不刷新会看不到启动横幅 - print(f"训练前端已启动：{url}", flush=True) - print(f" 配置契约：{url}api/train/schema", flush=True) - print(f" 状态快照：{url}api/train/state", flush=True) - print(f" 离线预览：{url}api/train/state?preview=1", flush=True) - print(" 界面只监听 127.0.0.1；Ctrl+C 退出并停止训练子进程。", flush=True) - if not args.no_browser: - try: - import webbrowser - webbrowser.open(url) - except Exception: - pass - try: - ui.serve_forever() - except KeyboardInterrupt: - pass - finally: - ui.shutdown() - return 0 - - -if __name__ == "__main__": # pragma: no cover - raise SystemExit(run_training_ui()) diff --git a/src/mcm26b/training_ui/static/app.js b/src/mcm26b/training_ui/static/app.js deleted file mode 100644 index 9b27cf9..0000000 --- a/src/mcm26b/training_ui/static/app.js +++ /dev/null @@ -1,841 +0,0 @@ -/* 训练前端主逻辑：纯原生 JS，无外部依赖。 - 职责分工： - - 表单由 /api/train/schema 的字段元数据生成，不在前端复制校验规则； - - 训练数值全部取自 /api/train/state 快照（服务端已算好进度、吞吐、 - 收敛判读），前端只做格式化与渲染，避免两套算法对不上； - - 指标增量拉取（after_metric 游标），日志按尾部行数拉取。 */ -"use strict"; - -const $ = (id) => document.getElementById(id); -const REFRESH_MS = 1500; -// 与服务端 MAX_METRICS 对齐：本地缓存上限，避免长时间训练把内存吃满 -const METRIC_LIMIT = 5000; - -const state = { - schema: null, - preview: false, // 离线预览：不再启动进程，只渲染示例快照 - polling: false, - saving: false, // 配置表单是否处于"训练中"锁定态 - metrics: [], // 本地累积的进度点（按 timesteps 递增） - metricCache: [], // metrics 的同一个数组（显式持有，避免 slice/concat 时丢引用） - metricNext: 0, // 本地已吃到的下一个下标（= 服务端下次应返回的 offset） - metricTotal: 0, // 服务端累计进度点总数（增量拉取的游标） - chartKey: "ep_rew_mean", - jobKey: null, // job_id 变化时清空本地缓存 - lastLogSignature: "", - lastStatus: null, // 最近一次快照的状态：决定暂停按钮是"暂停"还是"继续" - lastJob: null, // 最近一次快照：恢复按钮需要读 job.recovery -}; - -// ---------------------------------------------------------------- 工具 -function escapeHtml(value) { - return String(value == null ? "" : value) - .replace(/[&<>"']/g, (c) => ({ "&": "&", "<": "<", ">": ">", - '"': """, "'": "'" }[c])); -} -function fmtNumber(value, digits) { - if (value === null || value === undefined || value === "" || Number.isNaN(Number(value))) return "—"; - const num = Number(value); - const abs = Math.abs(num); - if (abs !== 0 && (abs < 1e-3 || abs >= 1e5)) return num.toExponential(2); - const d = digits === undefined ? (abs >= 100 ? 1 : abs >= 1 ? 2 : 4) : digits; - return num.toFixed(d); -} -function fmtInt(value) { - if (value === null || value === undefined || Number.isNaN(Number(value))) return "—"; - return Math.round(Number(value)).toLocaleString("en-US"); -} -function fmtDuration(seconds) { - if (seconds === null || seconds === undefined || !isFinite(seconds) || seconds < 0) return "—"; - const s = Math.round(seconds); - const h = Math.floor(s / 3600), m = Math.floor((s % 3600) / 60), sec = s % 60; - if (h > 0) return `${h}h${String(m).padStart(2, "0")}m`; - if (m > 0) return `${m}m${String(sec).padStart(2, "0")}s`; - return `${sec}s`; -} -function fmtClock(epochSeconds) { - if (!epochSeconds) return "—"; - const d = new Date(epochSeconds * 1000); - return `${String(d.getHours()).padStart(2, "0")}:${String(d.getMinutes()).padStart(2, "0")}:${String(d.getSeconds()).padStart(2, "0")}`; -} -function chip(id, label, value, tone) { - const el = $(id); - if (!el) return; - el.className = `chip ${tone || "gray"}`; - el.innerHTML = `${label}${escapeHtml(value)}`; -} -function setText(id, text) { const el = $(id); if (el) el.textContent = text; } - -// ---------------------------------------------------------------- 启动 -async function boot() { - try { - const res = await fetch("/api/train/schema"); - if (!res.ok) throw new Error(`HTTP ${res.status}`); - state.schema = await res.json(); - } catch (err) { - state.schema = { fields: [], groups: [], presets: [], metrics: [] }; - showBanner("bad", `无法读取配置契约（${err.message}）：请确认已通过 python -m mcm26b.training_ui 启动前端。`); - } - buildForm(); - await poll(); - bindChartInteractions(); - setInterval(poll, REFRESH_MS); -} - -function showBanner(tone, html) { - const el = $("banner"); - if (!html) { el.classList.add("hidden"); el.innerHTML = ""; return; } - el.className = `banner ${tone || ""}`; - el.innerHTML = html; - el.classList.remove("hidden"); -} - -// ---------------------------------------------------------------- 表单 -function buildForm() { - const presets = $("presets"); - presets.innerHTML = ""; - for (const preset of state.schema.presets || []) { - const btn = document.createElement("button"); - btn.type = "button"; - btn.className = "preset"; - btn.innerHTML = `${escapeHtml(preset.label)}${escapeHtml(preset.note || "")}`; - btn.onclick = () => { applyValues(preset.payload); $("form-error").textContent = ""; }; - presets.appendChild(btn); - } - - const wrap = $("form-groups"); - wrap.innerHTML = ""; - for (const group of state.schema.groups || []) { - const box = document.createElement("fieldset"); - box.className = "group"; - const fields = (state.schema.fields || []).filter((f) => f.group === group.key); - box.innerHTML = `${escapeHtml(group.label)}` + - `

${escapeHtml(group.hint || "")}

`; - for (const field of fields) box.appendChild(buildField(field)); - wrap.appendChild(box); - } - applyValues({}); -} - -function buildField(field) { - const row = document.createElement("div"); - row.className = "field"; - row.id = `field-${field.key}`; - const label = document.createElement("label"); - label.className = "flabel"; - label.innerHTML = `${escapeHtml(field.label)}` + - (field.unit ? `${escapeHtml(field.unit)}` : ""); - let control; - if (field.kind === "select") { - control = document.createElement("select"); - for (const choice of field.choices || []) { - const option = document.createElement("option"); - option.value = String(choice.value); - option.textContent = choice.label; - control.appendChild(option); - } - } else if (field.kind === "bool") { - control = document.createElement("input"); - control.type = "checkbox"; - row.classList.add("check"); - } else { - control = document.createElement("input"); - control.type = field.kind === "text" ? "text" : "number"; - if (field.kind === "number") control.step = field.step || "any"; - if (field.min !== undefined) control.min = field.min; - if (field.max !== undefined) control.max = field.max; - } - control.id = `cfg-${field.key}`; - control.dataset.key = field.key; - control.addEventListener("input", () => { row.classList.add("dirty"); row.classList.remove("invalid"); }); - row.appendChild(label); - row.appendChild(control); - if (field.help) { - const help = document.createElement("span"); - help.className = "fhelp"; - help.textContent = field.help; - row.appendChild(help); - } - return row; -} - -function setControl(field, value) { - const control = $(`cfg-${field.key}`); - if (!control) return; - if (field.kind === "bool") { control.checked = Boolean(value); return; } - if (value === null || value === undefined) { control.value = ""; return; } - control.value = String(value); -} - -function applyValues(values) { - for (const field of state.schema.fields || []) { - const has = Object.prototype.hasOwnProperty.call(values, field.key); - setControl(field, has ? values[field.key] : field.default); - const row = $(`field-${field.key}`); - if (row) row.classList.remove("dirty", "invalid"); - } -} - -function collectValues() { - const out = {}; - for (const field of state.schema.fields || []) { - const control = $(`cfg-${field.key}`); - if (!control) continue; - if (field.kind === "bool") { out[field.key] = control.checked; continue; } - const raw = String(control.value || "").trim(); - if (raw === "") { out[field.key] = null; continue; } - if (field.kind === "integer") out[field.key] = Number.parseInt(raw, 10); - else if (field.kind === "number") out[field.key] = Number.parseFloat(raw); - else if (field.kind === "select") { - const choices = field.choices || []; - const match = choices.find((c) => String(c.value) === raw); - out[field.key] = match ? match.value : raw; - } else out[field.key] = raw; - } - return out; -} - -/* 与后端 validate() 对齐的前置提示：只做提示，不阻止提交后端的权威判定 */ -function validateLocally(values) { - const problems = []; - for (const field of state.schema.fields || []) { - const value = values[field.key]; - if (value === null || value === undefined) continue; - if (field.kind === "integer" || field.kind === "number") { - if (!isFinite(value)) { problems.push(`${field.label} 不是有效数字`); markInvalid(field.key, true); continue; } - if (field.min !== undefined && value < field.min) { problems.push(`${field.label} 不能小于 ${field.min}`); markInvalid(field.key, true); continue; } - if (field.max !== undefined && value > field.max) { problems.push(`${field.label} 不能大于 ${field.max}`); markInvalid(field.key, true); continue; } - markInvalid(field.key, false); - } - } - if (values.rollout_steps && values.timesteps && values.rollout_steps > values.timesteps) { - problems.push("采样步数 n_steps 不能大于总步数 timesteps"); - markInvalid("rollout_steps", true); - } - if (values.batch_size && values.rollout_steps && values.envs - && values.batch_size > values.rollout_steps * values.envs) { - problems.push("批大小 batch_size 大于 n_steps × envs，PPO 每轮数据不足以分批"); - markInvalid("batch_size", true); - } - return problems; -} - -function markInvalid(key, bad) { - const row = $(`field-${key}`); - if (row) row.classList.toggle("invalid", Boolean(bad)); -} - -// ---------------------------------------------------------------- 轮询 -async function poll() { - if (state.polling) return; - if (state.preview) return; - state.polling = true; - try { - const query = new URLSearchParams({ - log_tail: $("log-tail").value || "400", - metric_limit: String(METRIC_LIMIT), - }); - if (state.metricTotal > 0) query.set("after_metric", String(state.metricTotal)); - const res = await fetch(`/api/train/state?${query.toString()}`); - const payload = await res.json(); - if (!res.ok) throw new Error(payload.error || `HTTP ${res.status}`); - chip("chip-conn", "连接", "已连接", "ok"); - render(payload.job); - } catch (err) { - chip("chip-conn", "连接", "断开", "bad"); - } finally { - state.polling = false; - } -} - -/* 合并一次快照返回的进度点。服务端用 metric_offset 给出这批点的绝对起点， - 客户端只需与"本地已吃到的下一个下标"（metricNext）对账： - - offset === metricNext：正常续接（增量拉取），concat； - - offset > metricNext：本地缓存与这批点之间缺了若干点（例如页面刷新后 - 服务端窗口已前移），保留本地旧点会让曲线出现空洞，因此整体替换； - - offset < metricNext：本地缓存比服务端更新（服务端重启/计数回退）， - 忽略这批过期点，不动本地缓存。 - 判据是"绝对下标"而不是长度：游标是服务端总数，本地可能被裁剪得更短， - 拿长度比较会把同一批点重复追加（本函数注释即为此 bug 的回归说明）。 - 下标必须显式转 Number：JSON 里若出现字符串形式的数字，JS 的 >= 会退化成 - 字典序比较（"14" >= "014" 为 false），静默走错分支。 */ -function absorbMetrics(job) { - const incoming = Array.isArray(job.metrics) ? job.metrics : []; - const rawOffset = Number(job.metric_offset); - const offset = Number.isFinite(rawOffset) ? rawOffset : null; - const cache = state.metricCache || []; - if (incoming.length && offset !== null) { - if (!cache.length || offset > state.metricNext) { - // 首批，或与本地之间有缺口（服务端窗口前移）：整体替换，避免曲线断档 - state.metricCache = incoming.slice(); - state.metricNext = offset + incoming.length; - } else if (offset === state.metricNext) { - state.metricCache = cache.concat(incoming); - state.metricNext = offset + incoming.length; - } else { - return; // 过期批次：本地已有更新的点，直接忽略 - } - } - if (state.metricCache.length > METRIC_LIMIT) { - state.metricCache = state.metricCache.slice(state.metricCache.length - METRIC_LIMIT); - } - state.metrics = state.metricCache; - const total = Number(job.metric_total); - if (Number.isFinite(total)) state.metricTotal = total; -} - -// ---------------------------------------------------------------- 渲染 -function render(job) { - if (!job) { - state.lastJob = null; - renderIdle(); - return; - } - state.lastJob = job; - if (job.job_id !== state.jobKey) { - state.jobKey = job.job_id; - state.metrics = []; - state.metricTotal = 0; - state.lastLogSignature = ""; - } - absorbMetrics(job); - - renderProcess(job); - renderConvergence(job); - renderCharts(); - renderLog(job); - renderEpisodes(job); - renderBanner(job); - renderRecovery(job); - renderJobLine(job); -} - -function renderIdle() { - chip("chip-process", "进程", "未启动", "gray"); - chip("chip-progress", "进度", "—", "gray"); - chip("chip-elapsed", "墙钟", "—", "gray"); - chip("chip-device", "设备", "—", "gray"); - setText("process-hint", "启动任务后显示子进程与步进进度"); - setText("steps-done", "0"); setText("steps-total", "0"); - setText("progress-percent", "0.0%"); - $("bar-fill").style.width = "0%"; - setText("eta", "—"); setText("eta-clock", "—"); setText("metric-total", "0"); - for (const id of ["stat-pid", "stat-fps", "stat-elapsed", "stat-rc"]) setText(id, "—"); - setText("stat-logs", "0"); - $("timeline").innerHTML = ""; - setText("command-preview", ""); - state.lastStatus = null; - const pauseBtn = $("btn-pause"); - pauseBtn.disabled = true; - pauseBtn.textContent = "暂停训练"; - chip("chip-converge", "判读", "等待数据", "gray"); - $("converge-notes").innerHTML = ""; - $("metric-grid").innerHTML = '

尚未启动训练，暂无参数。

'; - setFormLocked(false); -} - -function renderJobLine(job) { - const cfg = job.config || {}; - const parts = [ - `任务 ${job.job_id}`, `问题 ${cfg.problem}`, `阶段 ${cfg.stage}`, - `口径 ${cfg.reward_profile}/${cfg.config_profile}`, - `seed ${cfg.seed}`, `envs ${cfg.envs}·${cfg.vec_env}`, - ]; - setText("job-line", parts.join(" · ")); - chip("chip-device", "设备", `${cfg.device || "?"}`, "cyan"); -} - -function renderProcess(job) { - const process = job.process || {}; - const tone = process.tone || "gray"; - chip("chip-process", "进程", process.label || job.status, tone); - chip("chip-elapsed", "墙钟", fmtDuration(process.elapsed_s), tone === "gray" ? "gray" : "cyan"); - - const percent = process.percent; - setText("chip-progress-value", ""); - chip("chip-progress", "进度", - percent === null || percent === undefined ? "—" : `${percent.toFixed(1)}%`, - percent !== null && percent >= 100 ? "ok" : "cyan"); - setText("steps-done", fmtInt(process.trained_steps ?? process.done_steps)); - setText("steps-total", fmtInt(process.total_steps)); - setText("progress-percent", percent === null || percent === undefined ? "—" : `${percent.toFixed(2)}%`); - const fill = $("bar-fill"); - fill.style.width = `${percent === null || percent === undefined ? 0 : percent}%`; - fill.className = "bar-fill" + (tone === "bad" ? " bad" : tone === "warn" ? " warn" : tone === "ok" && percent >= 100 ? " ok" : ""); - setText("process-hint", process.hint || ""); - setText("eta", fmtDuration(process.remaining_s)); - setText("eta-clock", fmtClock(process.eta_epoch_s)); - setText("metric-total", fmtInt(process.metric_total)); - - setText("stat-pid", process.terminal ? "已退出" : (job.pid ? `PID ${job.pid}` : "—")); - setText("stat-pgid", process.terminal ? "已回收" : "独立会话"); - const fps = state.metrics.length ? lastFinite(state.metrics, "fps") : null; - setText("stat-fps", fps === null ? "—" : fmtNumber(fps, 1)); - setText("stat-elapsed", fmtDuration(process.elapsed_s)); - const rc = $("stat-rc"); - rc.textContent = job.returncode === null || job.returncode === undefined ? "—" : String(job.returncode); - rc.className = "v" + (job.returncode === 0 ? " ok" : job.returncode ? " bad" : ""); - setText("stat-logs", `${fmtInt(process.log_total)} / ${fmtInt(process.log_capacity)}`); - - renderTimeline(process); - setText("command-preview", job.command ? `$ ${job.command.join(" ")}` : ""); - state.lastStatus = job.status; - setFormLocked(job.status === "running" || job.status === "created" - || job.status === "paused" || job.status === "stopping"); - $("btn-stop").disabled = !(job.status === "running" || job.status === "created" - || job.status === "paused"); - const pauseBtn = $("btn-pause"); - pauseBtn.disabled = !(job.status === "running" || job.status === "paused"); - pauseBtn.textContent = job.status === "paused" ? "继续训练" : "暂停训练"; -} - -function renderTimeline(process) { - const wrap = $("timeline"); - const nodes = process.timeline || []; - if (!nodes.length) { wrap.innerHTML = ""; return; } - const active = nodes.findIndex((n) => n.state === "active"); - if (wrap.dataset.signature !== nodes.map((n) => n.label).join("|")) { - wrap.innerHTML = ""; - nodes.forEach((node, index) => { - if (index > 0) { - const link = document.createElement("span"); - link.className = "link"; - link.dataset.link = String(index); - wrap.appendChild(link); - } - const el = document.createElement("span"); - el.className = "node"; - el.dataset.node = node.key; - el.innerHTML = `${escapeHtml(node.label)}`; - wrap.appendChild(el); - }); - wrap.dataset.signature = nodes.map((n) => n.label).join("|"); - } - nodes.forEach((node, index) => { - const el = wrap.querySelector(`[data-node="${node.key}"]`); - if (!el) return; - let cls = "node"; - if (node.state === "done") cls += " done"; - else if (node.state === "active") cls += " active"; - if (node.state === "active" && process.tone === "bad") cls += " failed"; - el.className = cls; - const link = wrap.querySelector(`[data-link="${index}"]`); - if (link) link.className = "link" + (index <= active ? " done" : ""); - }); -} - -function setFormLocked(locked) { - state.saving = locked; - for (const field of state.schema.fields || []) { - const control = $(`cfg-${field.key}`); - if (control) control.disabled = locked; - } - $("btn-start").disabled = locked; - $("btn-start").textContent = locked ? "训练中…" : "开始训练"; -} - -function renderBanner(job) { - const process = job.process || {}; - if (state.preview) { - showBanner("warn", "离线预览模式：正在渲染示例快照，不会启动训练进程。点「刷新」可返回实时轮询。"); - return; - } - if (job.status === "failed") { - showBanner("bad", `训练异常结束（退出码 ${job.returncode === null ? "?" : job.returncode}）：${escapeHtml(job.error || "见日志尾部")}。`); - } else if (job.status === "paused") { - showBanner("warn", "训练已暂停：进程与显存仍在，但停在 step 边界不再推进。点「继续训练」从暂停处恢复，或点「停止训练」保存当前模型后退出。"); - } else if (job.status === "stopped") { - showBanner("warn", job.forced_kill - ? "停止请求超时，已强制结束进程树；最后一段模型可能未保存，请检查 checkpoint 目录。" - : "已按请求停止训练（控制文件协作式停止），脚本已在 step 边界保存当前模型与元数据。"); - } else if (job.status === "exited") { - showBanner("ok", "训练与评估流程正常结束，模型与元数据已写入输出目录。"); - } else if (process.remaining_s !== null && process.remaining_s !== undefined - && process.eta_epoch_s === null && job.status === "running") { - showBanner("warn", `尚未产出可估算吞吐的进度点，剩余时间按墙钟上限回退估计：${fmtDuration(process.remaining_s)}。`); - } else { - showBanner("", ""); - } -} - -// ---------------------------------------------------------------- 恢复 -function renderRecovery(job) { - const recovery = job.recovery || {}; - const box = $("recovery-box"); - if (!box) return; - if (!recovery.model || recovery.reason === "none") { - box.classList.add("hidden"); - return; - } - box.classList.remove("hidden"); - const steps = recovery.timesteps === null || recovery.timesteps === undefined - ? "步数未知" : `${fmtInt(recovery.timesteps)} 步`; - const when = recovery.updated_unix_s ? fmtClock(recovery.updated_unix_s) : "—"; - setText("recovery-text", - `可恢复产物：${steps} · 来源 ${recovery.reason || "?"} · 更新 ${when} · ${recovery.model}`); - const button = $("btn-recover"); - // 离线预览里的恢复信息是合成的，不允许真的去续训 - button.disabled = !recovery.can_resume || Boolean(state.preview); -} - -// ---------------------------------------------------------------- 收敛 -function renderConvergence(job) { - const convergence = job.convergence || {}; - chip("chip-converge", "判读", convergence.status || "等待数据", convergence.tone || "gray"); - const notes = convergence.notes || []; - $("converge-notes").innerHTML = notes.length - ? notes.map((note) => `
  • ${escapeHtml(note)}
    • `).join("") - : "
  • 暂无判读说明。
    • "; - - const grid = $("metric-grid"); - grid.innerHTML = ""; - const catalog = state.schema.metrics || []; - for (const metric of catalog) { - const points = seriesOf(metric.key); - const values = points.map((p) => p[1]); - const value = values.length ? values[values.length - 1] : null; - const prev = values.length > 1 ? values[values.length - 2] : null; - const delta = value !== null && prev !== null ? value - prev : null; - const card = document.createElement("div"); - card.className = "metric-card"; - const alert = isAlert(metric.key, value); - if (alert) card.classList.add("alert"); - const trendClass = delta === null || Math.abs(delta) < 1e-12 ? "flat" - : delta > 0 ? "up" : "down"; - const arrow = delta === null || Math.abs(delta) < 1e-12 ? "→" : delta > 0 ? "▲" : "▼"; - card.innerHTML = - `
    ${escapeHtml(metric.label)}` + - `${escapeHtml(fmtValue(metric, value))}
    ` + - `` + - `
    ${arrow} ${escapeHtml(delta === null ? "—" : fmtValue(metric, delta))}` + - `${points.length} 点
    `; - if (alert) card.title = alert; - grid.appendChild(card); - const canvas = card.querySelector("canvas"); - if (canvas && values.length >= 2) { - window.Charts.sparkline(canvas, values.slice(-120), { color: metric.color }); - } - } -} - -function isAlert(key, value) { - if (value === null || value === undefined) return null; - if (key === "approx_kl" && value > 0.05) return "近似 KL 超过 0.05：策略更新幅度过大"; - if (key === "explained_variance" && value < 0) return "解释方差为负：价值函数尚未拟合"; - if (key === "entropy_loss" && Math.abs(value) < 1e-3) return "策略熵接近 0：探索可能过早停止"; - return null; -} - -function fmtValue(metric, value) { - if (value === null || value === undefined || !isFinite(value)) return "—"; - if (metric.format === "integer") return fmtInt(value); - return fmtNumber(value); -} - -function seriesOf(key) { - const points = []; - for (const row of state.metrics) { - const value = Number(row[key]); - if (!isFinite(value)) continue; - const x = Number(row.timesteps ?? row.total_timesteps ?? points.length + 1); - points.push([isFinite(x) ? x : points.length + 1, value]); - } - return points; -} -function lastFinite(rows, key) { - for (let i = rows.length - 1; i >= 0; i--) { - const value = Number(rows[i][key]); - if (isFinite(value)) return value; - } - return null; -} - -// ---------------------------------------------------------------- 图表 -function renderCharts() { - const catalog = state.schema.metrics || []; - const tabs = $("chart-tabs"); - if (tabs.dataset.signature !== catalog.map((m) => m.key).join("|")) { - tabs.innerHTML = ""; - for (const metric of catalog) { - const btn = document.createElement("button"); - btn.type = "button"; - btn.className = "chart-tab"; - btn.dataset.key = metric.key; - btn.textContent = metric.label; - btn.onclick = () => { state.chartKey = metric.key; renderCharts(); }; - tabs.appendChild(btn); - } - tabs.dataset.signature = catalog.map((m) => m.key).join("|"); - } - for (const btn of tabs.querySelectorAll(".chart-tab")) { - btn.classList.toggle("active", btn.dataset.key === state.chartKey); - } - - const windowSize = Number($("chart-window").value || 0); - const primary = catalog.find((m) => m.role === "primary") || { key: "ep_rew_mean", label: "奖励", color: "#4f8cff" }; - const rewardPoints = slice(seriesOf(primary.key), windowSize); - const layout = window.Charts.line($("chart-reward"), { - series: [{ name: primary.label, points: rewardPoints, color: primary.color, fill: true }], - xFormat: (v) => `${fmtInt(v)}步`, - emptyText: "等待首个 rollout 的 ep_rew_mean……", - }); - setText("chart-title", primary.label); - setText("chart-sub", describeSeries(rewardPoints, primary)); - - const metric = catalog.find((m) => m.key === state.chartKey) || primary; - const metricPoints = slice(seriesOf(metric.key), windowSize); - const refs = []; - if (metric.key === "approx_kl") refs.push({ value: 0.05, color: "#f87171", label: "过猛阈值 0.05" }); - if (metric.key === "explained_variance") refs.push({ value: 0, color: "#93a1bd", label: "0（不如均值预测）" }); - const layout2 = window.Charts.line($("chart-metric"), { - series: [{ name: metric.label, points: metricPoints, color: metric.color, fill: true }], - refs: refs, - xFormat: (v) => `${fmtInt(v)}步`, - emptyText: `等待 ${metric.key} 数据……`, - }); - setText("chart2-title", metric.label); - setText("chart2-sub", describeSeries(metricPoints, metric)); - state.layouts = { reward: layout, metric: layout2, metricKey: metric.key, primary: primary }; -} - -function slice(points, size) { - if (!size || points.length <= size) return points; - return points.slice(points.length - size); -} - -function describeSeries(points, metric) { - if (!points.length) return "（无数据）"; - if (points.length === 1) return `仅 1 点 · 最新 ${fmtValue(metric, points[0][1])}`; - const first = points[0], last = points[points.length - 1]; - const delta = last[1] - first[1]; - const sign = delta > 0 ? "+" : ""; - return `${points.length} 点 · 窗口内 ${sign}${fmtValue(metric, delta)} · ` + - `最新 ${fmtValue(metric, last[1])} @ ${fmtInt(last[0])} 步`; -} - -function bindChartInteractions() { - for (const id of ["chart-reward", "chart-metric"]) { - const canvas = $(id); - if (!canvas) continue; - const overlay = document.createElement("canvas"); - overlay.className = "chart-overlay"; - overlay.style.width = "100%"; overlay.style.height = "100%"; - if (canvas.parentElement) { - canvas.parentElement.style.position = "relative"; - canvas.parentElement.appendChild(overlay); - } - const move = (event) => { - const show = $("chart-cursor").checked; - const rect = canvas.getBoundingClientRect(); - const which = id === "chart-reward" ? "reward" : "metric"; - const layout = state.layouts && state.layouts[which]; - const ctx = overlay.getContext("2d"); - const dpr = window.devicePixelRatio || 1; - if (overlay.width !== Math.round(rect.width * dpr) || overlay.height !== Math.round(rect.height * dpr)) { - overlay.width = Math.round(rect.width * dpr); - overlay.height = Math.round(rect.height * dpr); - } - ctx.setTransform(dpr, 0, 0, dpr, 0, 0); - ctx.clearRect(0, 0, rect.width, rect.height); - if (!layout || !show || !isFinite(layout.bounds.x0)) return; - window.Charts.drawCursor(overlay, layout, { - mouse: { x: event.clientX - rect.left, y: event.clientY - rect.top }, - xFormat: (v) => `${fmtInt(v)}步`, - }); - }; - canvas.addEventListener("mousemove", move); - canvas.addEventListener("mouseleave", () => { - const ctx = overlay.getContext("2d"); - ctx.clearRect(0, 0, overlay.width, overlay.height); - }); - } - $("chart-window").addEventListener("change", renderCharts); - $("chart-cursor").addEventListener("change", renderCharts); -} - -// ---------------------------------------------------------------- 日志 -function renderLog(job) { - const lines = job.log_lines || []; - const filter = ($("log-filter").value || "").trim().toLowerCase(); - const showJson = $("log-json").checked; - const errorsOnly = $("log-errors-only").checked; - const view = $("log-view"); - const signature = `${lines.length}|${job.log_total}|${filter}|${showJson}|${errorsOnly}|${lines[lines.length - 1] || ""}`; - if (signature === state.lastLogSignature) return; - state.lastLogSignature = signature; - - const follow = $("log-follow").checked; - const atBottom = view.scrollTop + view.clientHeight >= view.scrollHeight - 24; - const keep = follow || atBottom; - - const html = []; - let shown = 0; - for (let i = 0; i < lines.length; i++) { - const raw = lines[i]; - const lower = raw.toLowerCase(); - if (filter && !lower.includes(filter)) continue; - if (errorsOnly && !/error|traceback|failed|exception|错误|失败/.test(lower)) continue; - if (!showJson && raw.trim().startsWith("{") && raw.includes('"event"')) continue; - shown++; - const cls = classify(raw); - html.push(`${highlight(raw, filter)}`); - } - view.innerHTML = html.length ? html.join("") : '（无匹配日志行）'; - if (keep) view.scrollTop = view.scrollHeight; - setText("log-shown", `显示 ${shown} / 共 ${fmtInt(job.log_total)} 行`); - setText("log-meta", `stdout/stderr 合并捕获 · 服务端保留最近 ${fmtInt(job.process ? job.process.log_capacity : 4000)} 行`); -} - -function classify(line) { - const text = line.toLowerCase(); - if (/error|traceback|exception|failed|错误|失败/.test(text)) return "err"; - if (/warn|deprecat|警告/.test(text)) return "warn"; - if (line.trim().startsWith("{")) return "json"; - if (/^\s*[-=]{3,}|已保存|success|complete|评估|metadata/.test(text)) return "meta"; - if (!line.trim()) return "dim"; - return ""; -} - -function highlight(line, filter) { - const escaped = escapeHtml(line); - if (!filter) return escaped; - try { - return escaped.replace(new RegExp(`(${filter.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")})`, "gi"), - "$1"); - } catch (err) { return escaped; } -} - -// ---------------------------------------------------------------- 逐回合 -function renderEpisodes(job) { - const rows = job.monitor || []; - const tbody = $("episode-table").querySelector("tbody"); - const tail = rows.slice(-200).reverse(); - tbody.innerHTML = tail.map((row, index) => { - const seq = rows.length - index; - return `${seq}${fmtNumber(row.r)}` + - `${fmtNumber(row.l, 0)}${fmtNumber(row.t, 3)}`; - }).join("") || '尚无逐回合记录（Monitor CSV 随训练写入）'; -} - -// ---------------------------------------------------------------- 交互 -async function submitPayload(payload) { - const err = $("form-error"); - err.textContent = ""; - if (state.saving) return false; // 训练中表单锁定，续训按钮同样不许重复提交 - if (payload.resume && !payload.model) { - err.textContent = "续训需要填写模型输出路径"; - return false; - } - const res = await fetch("/api/train/start", { - method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify(payload), - }); - const body = await res.json().catch(() => ({})); - if (!res.ok) { - err.textContent = body.error || `启动失败（HTTP ${res.status}）`; - return false; - } - state.preview = false; - state.jobKey = null; - state.metrics = []; - state.metricCache = []; - state.metricNext = 0; - state.metricTotal = 0; - await poll(); - return true; -} - -$("config-form").addEventListener("submit", async (event) => { - event.preventDefault(); - if (state.saving) return; - const err = $("form-error"); - err.textContent = ""; - const values = collectValues(); - const problems = validateLocally(values); - if (problems.length) { err.textContent = problems[0]; return; } - const payload = {}; - for (const [key, value] of Object.entries(values)) { - if (value !== null) payload[key] = value; - } - await submitPayload(payload); -}); - -$("btn-recover").addEventListener("click", async () => { - const recovery = (state.lastJob && state.lastJob.recovery) || {}; - if (!recovery.can_resume || !recovery.model) return; - if (!confirm(`用最近的可恢复产物续训？\n${recovery.model}\n` - + "当前表单里的 timesteps 会作为**追加**步数。")) return; - // 同步回表单，用户随后可微调步数再手动点「开始训练」 - const modelField = $("cfg-model"); - if (modelField) modelField.value = recovery.model; - const resumeField = $("cfg-resume"); - if (resumeField) resumeField.checked = true; - const values = collectValues(); - values.model = recovery.model; - values.resume = true; - values.force = false; // --resume 与 --force 互斥 - await submitPayload(values); -}); - -$("btn-pause").addEventListener("click", async () => { - const resuming = state.lastStatus === "paused"; - const res = await fetch(resuming ? "/api/train/resume" : "/api/train/pause", { - method: "POST", headers: { "Content-Type": "application/json" }, body: "{}", - }); - const body = await res.json().catch(() => ({})); - if (!res.ok) { - showBanner("bad", `${resuming ? "继续" : "暂停"}失败：${escapeHtml(body.error || res.status)}`); - } else if (!resuming && body.paused === false) { - showBanner("warn", "当前没有可暂停的训练进程（可能已结束）。"); - } - await poll(); -}); - -$("btn-stop").addEventListener("click", async () => { - if (!confirm("确认停止当前训练？\n将通过控制文件请求优雅停止，脚本先保存当前模型与元数据再退出；" - + "若超时未响应则强制结束进程树。")) return; - const res = await fetch("/api/train/stop", { - method: "POST", headers: { "Content-Type": "application/json" }, body: "{}", - }); - const body = await res.json().catch(() => ({})); - if (!res.ok) showBanner("bad", `停止失败：${escapeHtml(body.error || res.status)}`); - await poll(); -}); - -$("btn-refresh").addEventListener("click", async () => { - state.preview = false; - showBanner("", ""); - await poll(); -}); - -$("btn-preview").addEventListener("click", async () => { - try { - const scenario = $("preview-scenario") ? $("preview-scenario").value : "rising"; - const res = await fetch(`/api/train/state?preview=1&scenario=${encodeURIComponent(scenario)}&log_tail=400`); - const payload = await res.json(); - if (!payload.job) throw new Error(payload.error || "服务端未返回示例快照"); - state.preview = true; - state.jobKey = null; - state.metrics = []; - state.metricCache = []; - state.metricNext = 0; - state.metricTotal = 0; - render(payload.job); - } catch (err) { - showBanner("bad", `离线预览不可用：${escapeHtml(err.message)}`); - } -}); - -$("btn-reset").addEventListener("click", () => { - applyValues({}); - $("form-error").textContent = ""; -}); - -for (const id of ["log-filter", "log-json", "log-errors-only", "log-tail"]) { - $(id).addEventListener("input", () => { - state.lastLogSignature = ""; - poll(); - }); -} - -boot(); diff --git a/src/mcm26b/training_ui/static/charts.js b/src/mcm26b/training_ui/static/charts.js deleted file mode 100644 index e34a8ce..0000000 --- a/src/mcm26b/training_ui/static/charts.js +++ /dev/null @@ -1,255 +0,0 @@ -/* 训练前端图表：原生 canvas 折线图，无外部依赖（离线可用，与评测前端同风格）。 - 只做三件事：坐标轴自适应、多序列折线、悬停读数；不引入缩放/图例等交互， - 因为训练曲线每 2 秒整体重绘一次，重量级图表库在这里只会增加风险。 - - 对外接口（挂到 globalThis.Charts）： - Charts.fit(canvas) 按 DPR 调整画布像素 - Charts.line(canvas, {series, refs, ...}) 折线图 - Charts.sparkline(canvas, values, opts) 指标卡片内的小趋势线 - 数据约定：series = [{ points: [[x, y], ...], color, width, fill }] */ -(function (global) { - "use strict"; - - var AXIS = "#2a3550", GRID = "rgba(42,53,80,.55)", DIM = "#93a1bd", TEXT = "#e6ecf8"; - var FONT = '11px ui-monospace, SFMono-Regular, Menlo, monospace'; - - function fit(canvas) { - var dpr = global.devicePixelRatio || 1; - var rect = canvas.getBoundingClientRect ? canvas.getBoundingClientRect() : null; - var cssWidth = (rect && rect.width) || canvas.clientWidth || canvas.width || 640; - var cssHeight = (rect && rect.height) || canvas.clientHeight || canvas.height || 200; - var w = Math.max(160, Math.round(cssWidth * dpr)); - var h = Math.max(80, Math.round(cssHeight * dpr)); - if (canvas.width !== w || canvas.height !== h) { canvas.width = w; canvas.height = h; } - var ctx = canvas.getContext("2d"); - ctx.setTransform(dpr, 0, 0, dpr, 0, 0); - return { ctx: ctx, w: w / dpr, h: h / dpr }; - } - - /* 第 0 层：数值格式（科学计数法只用于真正小而大的量级） */ - function fmt(value) { - if (value === null || value === undefined || Number.isNaN(value)) return "—"; - var abs = Math.abs(value); - if (abs !== 0 && (abs < 1e-3 || abs >= 1e5)) return value.toExponential(2); - if (abs >= 100) return value.toFixed(1); - if (abs >= 1) return value.toFixed(2); - if (abs >= 0.01) return value.toFixed(3); - return value.toFixed(4); - } - - function niceTicks(min, max, count) { - if (!isFinite(min) || !isFinite(max)) return [0, 1]; - if (min === max) { var pad = Math.abs(min) > 1e-9 ? Math.abs(min) * 0.1 : 0.5; - min -= pad; max += pad; } - var span = max - min; - var raw = span / Math.max(1, count); - var mag = Math.pow(10, Math.floor(Math.log10(raw))); - var norm = raw / mag; - var step = (norm <= 1 ? 1 : norm <= 2 ? 2 : norm <= 5 ? 5 : 10) * mag; - var start = Math.ceil(min / step) * step; - var ticks = []; - for (var v = start; v <= max + step * 1e-6 && ticks.length < 40; v += step) { - ticks.push(Math.abs(v) < step * 1e-9 ? 0 : v); - } - if (!ticks.length) ticks = [min, max]; - return ticks; - } - - function bounds(series) { - var xs = [], ys = []; - for (var i = 0; i < series.length; i++) { - var pts = series[i].points || []; - for (var j = 0; j < pts.length; j++) { - var p = pts[j]; - if (!p || !isFinite(p[0]) || !isFinite(p[1])) continue; - xs.push(p[0]); ys.push(p[1]); - } - } - if (!xs.length) return null; - var x0 = Math.min.apply(null, xs), x1 = Math.max.apply(null, xs); - var y0 = Math.min.apply(null, ys), y1 = Math.max.apply(null, ys); - if (x0 === x1) { x0 -= 0.5; x1 += 0.5; } - if (y0 === y1) { var pad = Math.abs(y0) > 1e-9 ? Math.abs(y0) * 0.08 : 0.5; - y0 -= pad; y1 += pad; } - return { x0: x0, x1: x1, y0: y0, y1: y1 }; - } - - /* ------------------------------------------------------------ 折线图 */ - function line(canvas, options) { - var opts = options || {}; - var axes = fit(canvas), ctx = axes.ctx, W = axes.w, H = axes.h; - var padL = opts.padLeft || 62, padR = 14, padT = 10, padB = 24; - ctx.clearRect(0, 0, W, H); - - var series = (opts.series || []).filter(function (s) { - return s && s.points && s.points.length; - }); - var box = bounds(series); - if (!box) { - ctx.fillStyle = DIM; ctx.font = FONT; - ctx.fillText(opts.emptyText || "等待数据……", padL, H / 2); - return null; - } - var yTicks = niceTicks(box.y0, box.y1, 4); - var yMin = Math.min(box.y0, yTicks[0]), yMax = Math.max(box.y1, yTicks[yTicks.length - 1]); - var plotW = Math.max(10, W - padL - padR), plotH = Math.max(10, H - padT - padB); - var sx = function (x) { return padL + (x - box.x0) / (box.x1 - box.x0) * plotW; }; - var sy = function (y) { return padT + (1 - (y - yMin) / (yMax - yMin)) * plotH; }; - - // 网格与刻度 - ctx.font = FONT; ctx.lineWidth = 1; - for (var i = 0; i < yTicks.length; i++) { - var y = sy(yTicks[i]); - ctx.strokeStyle = GRID; ctx.beginPath(); - ctx.moveTo(padL, y + 0.5); ctx.lineTo(padL + plotW, y + 0.5); ctx.stroke(); - ctx.fillStyle = DIM; ctx.textAlign = "right"; - ctx.fillText(fmt(yTicks[i]), padL - 6, y + 3.5); - } - ctx.textAlign = "center"; - for (var t = 0; t <= 4; t++) { - var xv = box.x0 + (box.x1 - box.x0) * (t / 4); - var x = sx(xv); - ctx.fillStyle = DIM; - ctx.fillText(opts.xFormat ? opts.xFormat(xv) : fmt(xv), x, H - 7); - } - ctx.textAlign = "left"; - ctx.strokeStyle = AXIS; ctx.beginPath(); - ctx.moveTo(padL + 0.5, padT); ctx.lineTo(padL + 0.5, padT + plotH); - ctx.lineTo(padL + plotW, padT + plotH); ctx.stroke(); - - // 参考线（例如 KL 的 0.05 阈值、解释方差的 0） - var refs = opts.refs || []; - for (var r = 0; r < refs.length; r++) { - var ref = refs[r]; - if (ref.value === undefined || ref.value < yMin || ref.value > yMax) continue; - var ry = sy(ref.value); - ctx.save(); - ctx.setLineDash([4, 4]); ctx.strokeStyle = ref.color || "#f59e0b"; - ctx.beginPath(); ctx.moveTo(padL, ry); ctx.lineTo(padL + plotW, ry); ctx.stroke(); - ctx.restore(); - if (ref.label) { - ctx.fillStyle = ref.color || "#f59e0b"; ctx.font = FONT; - ctx.fillText(ref.label, padL + 6, ry - 4); - } - } - - // 序列 - for (var s = 0; s < series.length; s++) { - var item = series[s]; - var pts = item.points; - ctx.strokeStyle = item.color || "#4f8cff"; - ctx.lineWidth = item.width || 1.6; - ctx.lineJoin = "round"; - ctx.beginPath(); - for (var k = 0; k < pts.length; k++) { - var px = sx(pts[k][0]), py = sy(pts[k][1]); - if (k === 0) ctx.moveTo(px, py); else ctx.lineTo(px, py); - } - ctx.stroke(); - if (item.fill && pts.length > 1) { - ctx.save(); - ctx.globalAlpha = 0.12; ctx.fillStyle = item.color || "#4f8cff"; - ctx.lineTo(sx(pts[pts.length - 1][0]), padT + plotH); - ctx.lineTo(sx(pts[0][0]), padT + plotH); - ctx.closePath(); ctx.fill(); ctx.restore(); - } - // 末点高亮：一眼看到最新值 - var last = pts[pts.length - 1]; - ctx.fillStyle = item.color || "#4f8cff"; - ctx.beginPath(); ctx.arc(sx(last[0]), sy(last[1]), 2.6, 0, Math.PI * 2); ctx.fill(); - } - - return { bounds: box, sx: sx, sy: sy, padL: padL, padT: padT, - plotW: plotW, plotH: plotH, yMin: yMin, yMax: yMax, series: series }; - } - - /* 悬停读数：把鼠标 x 映射到最接近的数据点，返回一次绘制的覆盖层信息 */ - function drawCursor(canvas, layout, options) { - var opts = options || {}; - if (!layout || !layout.series || !layout.series.length) return; - var axes = fit(canvas), ctx = axes.ctx; - var mouse = opts.mouse; - if (!mouse || mouse.x < layout.padL || mouse.x > layout.padL + layout.plotW) return; - var target = layout.bounds.x0 + - (mouse.x - layout.padL) / layout.plotW * (layout.bounds.x1 - layout.bounds.x0); - var lines = [], snapX = target; - for (var s = 0; s < layout.series.length; s++) { - var pts = layout.series[s].points; - if (!pts.length) continue; - // 数据已按 x 递增，二分定位最近点 - var lo = 0, hi = pts.length - 1, best = pts[hi]; - while (lo <= hi) { - var mid = (lo + hi) >> 1; - if (pts[mid][0] === target) { best = pts[mid]; break; } - if (pts[mid][0] < target) { best = pts[mid]; lo = mid + 1; } - else { hi = mid - 1; } - } - var next = pts[Math.min(pts.length - 1, lo)]; - if (next && Math.abs(next[0] - target) < Math.abs(best[0] - target)) best = next; - snapX = best[0]; - lines.push({ label: layout.series[s].name || "", color: layout.series[s].color, - value: best[1] }); - } - var cx = layout.sx(snapX); - ctx.save(); - ctx.setLineDash([3, 3]); ctx.strokeStyle = "rgba(230,236,248,.35)"; - ctx.beginPath(); ctx.moveTo(cx, layout.padT); - ctx.lineTo(cx, layout.padT + layout.plotH); ctx.stroke(); - ctx.restore(); - for (var i = 0; i < lines.length; i++) { - var item = lines[i]; - var cy = layout.sy(item.value); - ctx.fillStyle = item.color || "#4f8cff"; - ctx.beginPath(); ctx.arc(cx, cy, 3.2, 0, Math.PI * 2); ctx.fill(); - ctx.strokeStyle = "rgba(11,15,26,.9)"; ctx.lineWidth = 1.4; ctx.stroke(); - } - var text = (opts.xFormat ? opts.xFormat(snapX) : fmt(snapX)) + " · " + - lines.map(function (item) { return (item.label ? item.label + " " : "") + fmt(item.value); }) - .join(" "); - ctx.font = FONT; - var tw = ctx.measureText(text).width + 14; - var bx = Math.min(cx + 8, layout.padL + layout.plotW - tw); - bx = Math.max(layout.padL + 2, bx); - var by = layout.padT + 4; - ctx.fillStyle = "rgba(23,30,46,.94)"; ctx.strokeStyle = AXIS; - ctx.beginPath(); - if (ctx.roundRect) { ctx.roundRect(bx, by, tw, 20, 6); ctx.fill(); ctx.stroke(); } - else { ctx.fillRect(bx, by, tw, 20); ctx.strokeRect(bx, by, tw, 20); } - ctx.fillStyle = TEXT; ctx.textAlign = "left"; - ctx.fillText(text, bx + 7, by + 14); - } - - /* ------------------------------------------------------------ 迷你趋势线 */ - function sparkline(canvas, values, options) { - var opts = options || {}; - var axes = fit(canvas), ctx = axes.ctx, W = axes.w, H = axes.h; - ctx.clearRect(0, 0, W, H); - var clean = (values || []).filter(function (v) { return isFinite(v); }); - if (clean.length < 2) { - ctx.strokeStyle = GRID; ctx.beginPath(); - ctx.moveTo(0, H - 1.5); ctx.lineTo(W, H - 1.5); ctx.stroke(); - return; - } - var lo = Math.min.apply(null, clean), hi = Math.max.apply(null, clean); - if (lo === hi) { lo -= 0.5; hi += 0.5; } - var pad = 2; - var X = function (i) { return (i / (clean.length - 1)) * W; }; - var Y = function (v) { return H - pad - (v - lo) / (hi - lo) * (H - pad * 2); }; - ctx.strokeStyle = opts.color || "#4f8cff"; ctx.lineWidth = 1.4; - ctx.beginPath(); - for (var i = 0; i < clean.length; i++) { - if (i === 0) ctx.moveTo(X(i), Y(clean[i])); else ctx.lineTo(X(i), Y(clean[i])); - } - ctx.stroke(); - if (opts.fill !== false) { - ctx.save(); ctx.globalAlpha = 0.14; ctx.fillStyle = opts.color || "#4f8cff"; - ctx.lineTo(W, H); ctx.lineTo(0, H); ctx.closePath(); ctx.fill(); ctx.restore(); - } - var lastX = X(clean.length - 1), lastY = Y(clean[clean.length - 1]); - ctx.fillStyle = opts.color || "#4f8cff"; - ctx.beginPath(); ctx.arc(lastX - 1.5, lastY, 2.2, 0, Math.PI * 2); ctx.fill(); - } - - global.Charts = { fit: fit, line: line, drawCursor: drawCursor, - sparkline: sparkline, fmt: fmt }; -})(typeof globalThis !== "undefined" ? globalThis : this); diff --git a/src/mcm26b/training_ui/static/index.html b/src/mcm26b/training_ui/static/index.html deleted file mode 100644 index c1a93ac..0000000 --- a/src/mcm26b/training_ui/static/index.html +++ /dev/null @@ -1,180 +0,0 @@ - - - - - -模型训练前端 · CUMCM 2026 B HRL - - - -
    -
    -

    模型训练前端 HRL 高层 MaskablePPO

    -

    尚未启动训练任务

    -
    -
    - 进程未启动 - 进度— - 墙钟— - 设备— - 连接探测中 -
    -
    - - - - - -
    -
    - -
    - - - - - -
    -
    -

    执行进度与进程

    - 启动任务后显示子进程与步进进度 -
    - -
    -
    -
    - 0 / 0 环境步 - 0.0% -
    -
    -
    - 剩余估计 — - 预计结束 — - 进度点 0 -
    -
    -
    -
    子进程 PID—
    -
    进程组 / 会话独立
    -
    吞吐 FPS—
    -
    已用墙钟—
    -
    退出码—
    -
    日志行数0
    -
    -
    - -
    -

    -
    - - -
    -
    -

    训练配置

    - 字段与后端 TrainConfig 同源；留空字段取默认值 -
    -
    -
    -
    -
    - - - -
    -
    -
    - - -
    -
    -

    参数收敛状态

    - 判据为最近窗口的斜率与波动，仅供人工复核 -
    -
    - 判读等待数据 -
      -
      -
      - -
      -
      - - -
      -
      -
      -
      平均回合奖励
      - -
      -
      -
      近似 KL
      - -
      -
      -
      - - -
      -
      -

      实时运行日志

      - stdout/stderr 合并捕获 -
      -
      - - - - - - -
      - 
      （暂无输出）
      -
      - - -
      -
      -

      逐回合记录

      - 来自训练脚本写入的 Monitor CSV（每回合奖励/长度/时刻） -
      -
      - - - -
      #奖励 r长度 l时刻 t (s)
      -
      -
      -
      - -
      - 本地回环训练前端：与评测前端共用「白名单静态资源 + JSON 轮询 API」模式； - 命令由服务端以参数列表构造（不经过 shell），停止时向独立进程组发送 SIGTERM。 -
      - - - - - diff --git a/src/mcm26b/training_ui/static/style.css b/src/mcm26b/training_ui/static/style.css deleted file mode 100644 index 185d01d..0000000 --- a/src/mcm26b/training_ui/static/style.css +++ /dev/null @@ -1,230 +0,0 @@ -/* 训练前端样式：沿用评测前端（referee/static/style.css）的暗色变量与圆角面板 - 语言，保证两个界面像同一套系统的两个页面；本页信息密度更高： - 进度条、进程时间线、参数卡片网格、曲线与日志终端。 */ - -:root { - --bg: #0f1420; --panel: #171e2e; --panel2: #1d2740; --line: #2a3550; - --text: #e6ecf8; --dim: #93a1bd; --accent: #4f8cff; --ok: #34d399; - --warn: #f59e0b; --bad: #f87171; --cyan: #38bdf8; --gray: #6b7280; - --mono: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; -} -* { box-sizing: border-box; } -body { - margin: 0; background: var(--bg); color: var(--text); - font: 14px/1.5 "PingFang SC", "Microsoft YaHei", system-ui, sans-serif; -} -code, .mono { background: var(--panel2); padding: 1px 6px; border-radius: 4px; color: var(--cyan); font-family: var(--mono); font-size: 12px; } -main { max-width: 1240px; margin: 0 auto; padding: 16px 22px 48px; } -footer { color: var(--dim); font-size: 12px; text-align: center; padding: 14px 22px; border-top: 1px solid var(--line); } -.hidden { display: none !important; } -.hint { color: var(--dim); font-size: 12px; } -h2 { font-size: 15px; margin: 0; } - -/* ------------------------------------------------------------ 顶栏 */ -.topbar { - position: sticky; top: 0; z-index: 20; - display: flex; align-items: center; gap: 18px; flex-wrap: wrap; - padding: 12px 22px; border-bottom: 1px solid var(--line); - background: rgba(23,30,46,.96); backdrop-filter: blur(6px); -} -.brand h1 { font-size: 17px; margin: 0; } -.brand .sub { font-size: 12px; color: var(--dim); font-weight: normal; margin-left: 8px; } -.brand .tagline { margin: 3px 0 0; color: var(--dim); font-size: 12px; font-family: var(--mono); } -.topstatus { display: flex; gap: 8px; flex-wrap: wrap; margin-left: auto; } -.topactions { display: flex; gap: 8px; } - -.chip { - background: var(--panel2); border: 1px solid var(--line); border-radius: 999px; - padding: 3px 12px; font-size: 12px; color: var(--dim); white-space: nowrap; -} -.chip b { color: var(--text); font-weight: 600; margin-left: 4px; } -.chip.gray b { color: var(--gray); } -.chip.ok b { color: var(--ok); } -.chip.warn b { color: var(--warn); } -.chip.bad b { color: var(--bad); } -.chip.cyan b { color: var(--cyan); } - -button, .button { - background: var(--panel2); color: var(--text); border: 1px solid var(--line); - border-radius: 6px; padding: 7px 14px; font-size: 13px; cursor: pointer; -} -button:hover:enabled, .button:hover { border-color: var(--accent); } -button:disabled { opacity: .45; cursor: not-allowed; } -button.primary { background: var(--accent); border-color: var(--accent); color: #fff; } -button.primary:hover:enabled { filter: brightness(1.08); } -button.danger { background: transparent; border-color: var(--bad); color: var(--bad); } -button.danger:hover:enabled { background: rgba(248,113,113,.12); } - -.banner { - border: 1px solid var(--bad); border-left: 4px solid var(--bad); - background: rgba(248,113,113,.08); border-radius: 8px; - padding: 10px 14px; margin: 14px 0 0; font-size: 13px; -} -.banner.ok { border-color: var(--ok); border-left-color: var(--ok); background: rgba(52,211,153,.08); } -.banner.warn { border-color: var(--warn); border-left-color: var(--warn); background: rgba(245,158,11,.08); } -.banner b { font-family: var(--mono); } - -/* 恢复条：显示最近一份完整落盘的产物，并提供一键续训 */ -.recovery { - display: flex; align-items: center; justify-content: space-between; gap: 12px; - border: 1px solid var(--cyan, #22d3ee); border-left: 4px solid var(--cyan, #22d3ee); - background: rgba(34,211,238,.07); border-radius: 8px; - padding: 8px 12px; margin: 10px 0 0; font-size: 13px; -} -.recovery .hint { overflow-wrap: anywhere; } - -/* ------------------------------------------------------------ 面板 */ -.panel { - border: 1px solid var(--line); border-radius: 12px; background: var(--panel); - padding: 14px 16px; margin-top: 16px; -} -.panel-head { display: flex; align-items: baseline; gap: 12px; margin-bottom: 12px; flex-wrap: wrap; } -.panel-head h2 { color: var(--text); } -.panel-head h2::before { - content: ""; display: inline-block; width: 3px; height: 13px; margin-right: 8px; - background: var(--accent); border-radius: 2px; vertical-align: -1px; -} - -/* ------------------------------------------------------------ 进度 */ -.progress-row { display: flex; gap: 20px; align-items: stretch; flex-wrap: wrap; } -.progress-main { flex: 1 1 380px; min-width: 320px; } -.progress-labels { display: flex; justify-content: space-between; font-size: 13px; margin-bottom: 6px; } -.progress-labels b { font-family: var(--mono); font-size: 15px; } -.bar { height: 12px; border-radius: 999px; background: #0b0f1a; border: 1px solid var(--line); overflow: hidden; } -.bar-fill { - height: 100%; width: 0%; border-radius: 999px; - background: linear-gradient(90deg, #2f6fe0, var(--accent) 60%, var(--cyan)); - transition: width .3s ease; -} -.bar-fill.warn { background: linear-gradient(90deg, #b45309, var(--warn)); } -.bar-fill.bad { background: linear-gradient(90deg, #b91c1c, var(--bad)); } -.bar-fill.ok { background: linear-gradient(90deg, #059669, var(--ok)); } -.progress-foot { display: flex; gap: 18px; margin-top: 7px; flex-wrap: wrap; } - -.stat-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(132px, 1fr)); gap: 8px; flex: 1 1 420px; } -.stat { - border: 1px solid var(--line); border-radius: 8px; background: var(--panel2); - padding: 7px 10px; display: flex; flex-direction: column; gap: 2px; -} -.stat .k { color: var(--dim); font-size: 11px; } -.stat .v { font-family: var(--mono); font-size: 15px; } -.stat .v.ok { color: var(--ok); } -.stat .v.warn { color: var(--warn); } -.stat .v.bad { color: var(--bad); } - -.timeline { display: flex; align-items: center; gap: 0; margin-top: 16px; flex-wrap: wrap; } -.node { display: flex; align-items: center; gap: 8px; } -.node .dot { - width: 12px; height: 12px; border-radius: 50%; border: 2px solid var(--line); - background: var(--panel2); flex: none; -} -.node .txt { font-size: 12px; color: var(--dim); } -.node.done .dot { border-color: var(--ok); background: var(--ok); } -.node.done .txt { color: var(--text); } -.node.active .dot { border-color: var(--accent); background: var(--accent); box-shadow: 0 0 0 4px rgba(79,140,255,.18); } -.node.active .txt { color: var(--accent); font-weight: 600; } -.node.failed .dot { border-color: var(--bad); background: var(--bad); } -.node.failed .txt { color: var(--bad); } -.link { width: 34px; height: 2px; background: var(--line); flex: none; margin: 0 8px; } -.link.done { background: var(--ok); } -#command-preview { margin: 12px 0 0; font-family: var(--mono); word-break: break-all; line-height: 1.7; } - -/* ------------------------------------------------------------ 表单 */ -.presets { display: flex; flex-wrap: wrap; gap: 8px; margin-bottom: 14px; } -.preset { - border: 1px solid var(--line); border-radius: 8px; background: var(--panel2); - padding: 7px 12px; font-size: 12px; cursor: pointer; color: var(--text); -} -.preset:hover { border-color: var(--accent); } -.preset .note { display: block; color: var(--dim); font-size: 11px; margin-top: 2px; } -.form-groups { display: grid; grid-template-columns: repeat(auto-fit, minmax(268px, 1fr)); gap: 14px; } -fieldset.group { border: 1px solid var(--line); border-radius: 10px; padding: 10px 12px 12px; margin: 0; background: #141b29; } -fieldset.group legend { color: var(--dim); font-size: 12px; padding: 0 6px; } -fieldset.group .ghint { color: var(--dim); font-size: 11px; margin: 0 0 8px; } -.field { display: flex; flex-direction: column; gap: 4px; margin-bottom: 10px; } -.field .flabel { font-size: 12px; color: var(--dim); display: flex; gap: 6px; align-items: baseline; } -.field .flabel .unit { color: var(--gray); font-size: 11px; } -.field input[type="text"], .field input[type="number"], .field select { - background: var(--panel2); border: 1px solid var(--line); color: var(--text); - border-radius: 6px; padding: 6px 9px; font-size: 13px; width: 100%; - font-family: var(--mono); -} -.field input:focus, .field select:focus, .log-toolbar input:focus { outline: none; border-color: var(--accent); } -.field .fhelp { font-size: 11px; color: var(--gray); } -.field.check { flex-direction: row; align-items: center; gap: 8px; } -.field.check input { width: auto; } -.field.dirty input, .field.dirty select { border-color: rgba(79,140,255,.65); } -.field.invalid input, .field.invalid select { border-color: var(--bad); } -.form-foot { display: flex; align-items: center; gap: 12px; margin-top: 14px; flex-wrap: wrap; } -.form-error { color: var(--bad); font-size: 12px; } - -/* ------------------------------------------------------------ 收敛 */ -.converge-head { display: flex; gap: 14px; align-items: flex-start; flex-wrap: wrap; margin-bottom: 12px; } -.notes { margin: 0; padding-left: 18px; color: var(--dim); font-size: 12px; max-width: 760px; } -.notes li { margin: 2px 0; } -.metric-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(186px, 1fr)); gap: 10px; } -.metric-card { - border: 1px solid var(--line); border-radius: 10px; background: var(--panel2); - padding: 9px 11px; display: flex; flex-direction: column; gap: 5px; -} -.metric-card .mhead { display: flex; justify-content: space-between; align-items: baseline; gap: 8px; } -.metric-card .mname { font-size: 12px; color: var(--dim); } -.metric-card .mvalue { font-family: var(--mono); font-size: 17px; } -.metric-card .mfoot { display: flex; justify-content: space-between; font-size: 11px; color: var(--gray); gap: 6px; } -.metric-card canvas { width: 100%; height: 40px; display: block; } -.trend.up { color: var(--ok); } .trend.down { color: var(--warn); } -.trend.flat { color: var(--dim); } .trend.none { color: var(--gray); } -.metric-card.alert { border-color: rgba(248,113,113,.55); } -.metric-card.alert .mvalue { color: var(--bad); } - -/* ------------------------------------------------------------ 图表 */ -.chart-toolbar { display: flex; align-items: center; gap: 14px; flex-wrap: wrap; margin: 16px 0 8px; } -.chart-tabs { display: flex; gap: 6px; flex-wrap: wrap; } -.chart-tab { - border: 1px solid var(--line); background: var(--transparent, transparent); - color: var(--dim); border-radius: 999px; padding: 4px 12px; font-size: 12px; -} -.chart-tab.active { border-color: var(--accent); color: var(--text); background: var(--panel2); } -label.inline { font-size: 12px; color: var(--dim); display: flex; gap: 6px; align-items: center; } -label.inline input[type="checkbox"] { width: auto; } -label.inline select, .log-toolbar select { - background: var(--panel2); color: var(--text); border: 1px solid var(--line); - border-radius: 6px; padding: 4px 8px; font-size: 12px; -} -.chart-stack { display: flex; flex-direction: column; gap: 14px; } -.chart-figure { margin: 0; border: 1px solid var(--line); border-radius: 10px; background: #0b0f1a; padding: 10px 12px 4px; } -.chart-figure figcaption { display: flex; gap: 10px; align-items: baseline; font-size: 12px; margin-bottom: 6px; } -.chart-figure figcaption b { color: var(--text); font-size: 13px; } -.chart-figure canvas { width: 100%; display: block; } - -/* ------------------------------------------------------------ 日志 */ -.log-toolbar { display: flex; align-items: center; gap: 12px; flex-wrap: wrap; margin-bottom: 8px; } -.log-toolbar input[type="search"] { - background: var(--panel2); border: 1px solid var(--line); color: var(--text); - border-radius: 6px; padding: 6px 10px; font-size: 12px; min-width: 240px; -} -.log-view { - margin: 0; background: #0b0f1a; border: 1px solid var(--line); border-radius: 10px; - padding: 10px 12px; height: 320px; overflow: auto; font-family: var(--mono); - font-size: 12px; line-height: 1.55; white-space: pre-wrap; word-break: break-word; -} -.log-view .l { display: block; } -.log-view .l.err { color: var(--bad); } -.log-view .l.warn { color: var(--warn); } -.log-view .l.json { color: var(--cyan); } -.log-view .l.meta { color: var(--ok); } -.log-view .l.dim { color: var(--gray); } -.log-view mark { background: rgba(245,158,11,.28); color: inherit; border-radius: 2px; } - -/* ------------------------------------------------------------ 表格 */ -.table-wrap { border: 1px solid var(--line); border-radius: 10px; overflow: auto; max-height: 300px; } -table { width: 100%; border-collapse: collapse; font-size: 12px; } -th, td { text-align: left; padding: 6px 10px; border-bottom: 1px solid var(--line); white-space: nowrap; } -th { position: sticky; top: 0; background: var(--panel2); color: var(--dim); font-weight: 500; } -td { font-family: var(--mono); } -tbody tr:hover { background: rgba(79,140,255,.06); } - -@media (max-width: 760px) { - .topstatus { margin-left: 0; } - .progress-row { gap: 12px; } -} diff --git a/tests/js/training_ui_harness.cjs b/tests/js/training_ui_harness.cjs deleted file mode 100644 index 0b773e0..0000000 --- a/tests/js/training_ui_harness.cjs +++ /dev/null @@ -1,285 +0,0 @@ -/* 训练前端 JS 逻辑回归（由 pytest 经 node 调起，见 tests/test_training_ui_js.py）。 - 以 DOM 桩加载发布的 charts.js / app.js 原文，用服务端真实生成的示例快照 - （services/demo.py 的 demo_snapshot JSON）驱动渲染，断言： - 1) 表单由 schema 生成、字段数与后端契约一致； - 2) 指标卡/曲线/日志/逐回合表都拿到数据并渲染，无异常抛出； - 3) 三种收敛场景的判读文案正确落到顶部 chip； - 4) 增量指标游标合并不丢点也不重点（曾是该前端的典型 bug）； - 5) charts.js 的刻度与格式函数在极端输入下不抛异常。 - 曾回归出两个"加载即白屏"隐患：job.process 缺失时直接取下标、以及游标 - 与 metric_limit 同时生效时重复追加同批进度点。本文件防其复发。 */ -"use strict"; -const fs = require("fs"); -const path = require("path"); - -const REPO = process.env.REPO; -if (!REPO) throw new Error("REPO env required"); -const STATIC = path.join(REPO, "src", "mcm26b", "training_ui", "static"); -const DEMO = process.env.DEMO_STATE; -if (!DEMO) throw new Error("DEMO_STATE env required"); - -const scenarios = JSON.parse(fs.readFileSync(DEMO, "utf8")); -const schema = JSON.parse(fs.readFileSync(process.env.SCHEMA, "utf8")); - -// ---------------------------------------------------------------- DOM 桩 -/* className 是唯一真源：代码里会直接赋值 className（见 chip()），也会用 - classList.add/toggle；如果 classList 自带一个独立 Set，两者就会不同步， - 测试会误报。这里让 classList 每次读写都落回 className。 */ -function makeClassList(el) { - const read = () => String(el.className || "").split(/\s+/).filter(Boolean); - const write = (list) => { el.className = [...new Set(list)].join(" "); }; - return { - add: (...c) => write(read().concat(c)), - remove: (...c) => write(read().filter((x) => !c.includes(x))), - toggle: (c, force) => { - const on = force === undefined ? !read().includes(c) : !!force; - write(on ? read().concat(c) : read().filter((x) => x !== c)); - return on; - }, - contains: (c) => read().includes(c), - }; -} -const ctxStub = new Proxy({}, { - get(t, prop) { - if (!(prop in t)) { - t[prop] = typeof prop === "string" && - /^(arc|beginPath|closePath|fill|stroke|moveTo|lineTo|fillText|strokeText|strokeRect|fillRect|clearRect|setLineDash|setTransform|save|restore|roundRect|measureText|scale|translate)$/.test(prop) - ? (prop === "measureText" ? () => ({ width: 40 }) : () => {}) - : function () {}; - } - return t[prop]; - }, - set(t, prop, v) { t[prop] = v; return true; }, -}); -const allElements = []; -/* querySelectorAll 桩：支持 app.js 实际用到的选择器 - [data-node="x"] / [data-link="n"] → 时间线节点与连接线 - .chart-tab → 曲线页签（用于切换 active） - tbody → 逐回合表体 - 子元素按 tag 记账，因此 page.querySelector("tbody") 每次返回同一个对象。 */ -function matchesSelector(el, sel) { - const attr = /^\[data-([a-z]+)="?([^"\]]*)"?\]$/.exec(sel); - if (attr) return el.dataset && String(el.dataset[attr[1]]) === attr[2]; - if (sel.startsWith(".")) return String(el.className || "").split(/\s+/).includes(sel.slice(1)); - if (el.tag === sel) return true; - const byTag = el.byTag && el.byTag[sel]; - return Boolean(byTag); -} -function makeEl(id, tag) { - const el = { - id, tag: tag || id, textContent: "", innerHTML: "", value: "", checked: true, - disabled: false, className: "", - files: [], width: 600, height: 240, style: {}, handlers: {}, children: [], - byTag: {}, dataset: {}, scrollTop: 0, scrollHeight: 100, clientHeight: 100, - get classList() { return makeClassList(this); }, - addEventListener(type, fn) { (this.handlers[type] ||= []).push(fn); }, - removeEventListener() {}, click() {}, matches() { return false; }, - appendChild(child) { this.children.push(child); return child; }, - removeChild(child) { this.children = this.children.filter((c) => c !== child); }, - setAttribute() {}, focus() {}, - querySelector(sel) { return this.querySelectorAll(sel)[0] || this.childFor(sel); }, - // 表单里的静态子元素（如 ）不在 children 里，按需惰性创建并缓存 - childFor(sel) { - if (!this.byTag[sel]) this.byTag[sel] = makeEl(`${id}:${sel}`, sel); - return this.byTag[sel]; - }, - querySelectorAll(sel) { - const found = []; - const walk = (node) => { - for (const child of node.children || []) { - if (matchesSelector(child, sel)) found.push(child); - walk(child); - } - }; - walk(this); - return found; - }, - getContext() { return ctxStub; }, - getBoundingClientRect() { return { left: 0, top: 0, width: 600, height: 240 }; }, - }; - allElements.push(el); - return el; -} -const els = new Map(); -const $get = (id) => { if (!els.has(id)) els.set(id, makeEl(id)); return els.get(id); }; -// 几个会被读取初值的控件（否则 value 为空字符串，Number("") === 0） -for (const [id, value] of [["log-tail", "400"], ["chart-window", "120"]]) $get(id).value = value; -$get("chart-cursor").checked = true; -$get("log-follow").checked = true; -$get("log-json").checked = true; -$get("log-errors-only").checked = false; // 只保留"显示进度 JSON"这一项过滤 -$get("preview-scenario").value = "rising"; - -globalThis.requestAnimationFrame = (cb) => { cb(0); return 1; }; -globalThis.cancelAnimationFrame = () => {}; -globalThis.performance = { now: () => Date.now() }; -globalThis.setInterval = () => 0; -globalThis.location = { port: "8036" }; -globalThis.confirm = () => true; -globalThis.alert = () => {}; -globalThis.addEventListener = () => {}; -globalThis.devicePixelRatio = 2; -globalThis.window = globalThis; -globalThis.document = { - getElementById: $get, - createElement: (tag) => makeEl(tag), - addEventListener() {}, -}; -// 元素挂载计数：断言"指标卡确实被创建"而不是静默空转 -let createdCount = 0; -const realCreate = globalThis.document.createElement; -globalThis.document.createElement = (tag) => { createdCount++; return realCreate(tag); }; - -globalThis.fetch = async (url) => { - const json = async () => { - if (url.startsWith("/api/train/schema")) return schema; - if (url.includes("state")) { - const match = /scenario=([a-z-]+)/.exec(url); - const key = match ? match[1] : "rising"; - return { job: scenarios[key] }; - } - throw new Error("unexpected fetch " + url); - }; - return { ok: true, status: 200, json }; -}; - -// ---------------------------------------------------------------- 装载发布代码 -const chartsSrc = fs.readFileSync(path.join(STATIC, "charts.js"), "utf8"); -const appSrc = fs.readFileSync(path.join(STATIC, "app.js"), "utf8"); -new Function(chartsSrc)(); -const appApi = new Function( - appSrc + "\n;return { seriesOf, classify, highlight, absorbMetrics, state, render, fmtNumber, fmtDuration };" -)(); -// app.js 末尾会自动 boot()，这里等它把 schema/首次快照渲染完 -const flush = async () => { for (let i = 0; i < 40; i++) await new Promise((r) => setImmediate(r)); }; -function assert(cond, message) { if (!cond) throw new Error(message); } -function stripTags(html) { return String(html).replace(/<[^>]*>/g, ""); } - -(async () => { - await flush(); - - // 1) schema 驱动表单 - const groups = $get("form-groups").children; - assert(groups.length === schema.groups.length, - `表单分组数应为 ${schema.groups.length}，实际 ${groups.length}`); - const cfgField = $get("cfg-timesteps"); - assert(cfgField && String(cfgField.value) === String(schema.fields.find((f) => f.key === "timesteps").default), - `默认值未写入表单: ${cfgField && cfgField.value}`); - assert($get("presets").children.length === schema.presets.length, "预设按钮数不匹配"); - - // 2) 首帧（rising 场景）渲染 - const state = appApi.state; - assert(state.metrics.length > 0, "首次快照后本地进度点为空"); - const chips = $get("chip-converge").innerHTML; - assert(chips.includes(scenarios.rising.convergence.status), - `收敛 chip 未渲染: ${chips}`); - assert($get("stat-pid").textContent === "PID 42424", `PID 未渲染: ${$get("stat-pid").textContent}`); - assert($get("metric-grid").children.length === schema.metrics.length, - `指标卡数量应为 ${schema.metrics.length}，实际 ${$get("metric-grid").children.length}`); - assert($get("chart-tabs").children.length === schema.metrics.length, - "曲线页签数量与指标字典不一致"); - assert(stripTags($get("log-view").innerHTML).includes("progress"), - "日志区没有渲染进度行"); - assert($get("episode-table").querySelector("tbody") !== null, "逐回合表缺少 tbody"); - const tbody = $get("episode-table").querySelector("tbody"); - assert(tbody.innerHTML.includes(""), - `逐回合表未渲染任何行: monitor=${(scenarios.rising.monitor || []).length}`); - assert(Number($get("steps-total").textContent.replace(/,/g, "")) === scenarios.rising.config.timesteps, - `总步数渲染异常: ${$get("steps-total").textContent}`); - - // 3) 三种收敛场景切到对应 chip - const tones = { rising: "ok", plateau: "ok", "kl-spike": "bad" }; - for (const key of Object.keys(scenarios)) { - const rendered = appApi.render(scenarios[key]); - void rendered; - const html = $get("chip-converge").innerHTML; - assert(html.includes(scenarios[key].convergence.status), - `场景 ${key} 的判读未落到 chip: ${html}`); - assert($get("chip-converge").classList.contains(tones[key]), - `场景 ${key} 的色调应为 ${tones[key]}`); - } - assert(String($get("chart2-title").textContent).length > 0, "第二张曲线标题为空"); - - // 4) 增量游标合并：续接 / 有缺口 / 过期批次 三条分支 - const job = JSON.parse(JSON.stringify(scenarios.rising)); - const total = job.metrics.length; - const reset = () => { - appApi.state.metrics = []; - appApi.state.metricCache = []; - appApi.state.metricNext = 0; - appApi.state.metricTotal = 0; - }; - const feed = (metrics, offset) => appApi.absorbMetrics( - { ...job, metrics, metric_offset: offset, metric_total: total }); - const stamp = (i) => job.metrics[i].timesteps; - - reset(); - feed(job.metrics.slice(0, 10), 0); - assert(appApi.state.metrics.length === 10, `首批应 10 点，实际 ${appApi.state.metrics.length}`); - feed(job.metrics.slice(10, 14), 10); - assert(appApi.state.metrics.length === 14, `续接后应 14 点，实际 ${appApi.state.metrics.length}`); - assert(appApi.state.metricNext === 14, `metricNext 应为 14，实际 ${appApi.state.metricNext}`); - feed([], 14); - assert(appApi.state.metrics.length === 14, `无新点不得改变长度，实际 ${appApi.state.metrics.length}`); - - // 有缺口（服务端窗口前移）：整体替换，避免曲线出现空洞 - feed(job.metrics.slice(20, 30), 20); - assert(appApi.state.metrics.length === 10, `有缺口时应替换为 10 点，实际 ${appApi.state.metrics.length}`); - assert(appApi.state.metrics[0].timesteps === stamp(20), "替换后起点错位"); - assert(appApi.state.metricNext === 30, `游标未随替换更新: ${appApi.state.metricNext}`); - - // 过期批次（服务端计数回退）：忽略，本地不动 - feed(job.metrics.slice(0, 5), 0); - assert(appApi.state.metrics.length === 10, `过期批次应被忽略，实际 ${appApi.state.metrics.length}`); - assert(appApi.state.metrics[0].timesteps === stamp(20), "过期批次污染了本地缓存"); - - // 字符串形式的 offset：必须按数值比较，不得退化成字典序 - reset(); - feed(job.metrics.slice(0, 10), "0"); - feed(job.metrics.slice(10, 14), "10"); - assert(appApi.state.metrics.length === 14, - `字符串 offset 应正常续接为 14 点，实际 ${appApi.state.metrics.length}`); - - // 缺失 offset（旧版服务端）：退化为按到达顺序追加，不得丢点 - reset(); - appApi.absorbMetrics({ ...job, metrics: job.metrics.slice(0, 6) }); - assert(appApi.state.metrics.length === 6, `缺 offset 时首批应 6 点，实际 ${appApi.state.metrics.length}`); - - // 5) 日志分类与高亮 - assert(appApi.classify("Traceback (most recent call last):") === "err", "Traceback 未归入错误行"); - assert(appApi.classify("UserWarning: x") === "warn", "警告未归入 warn"); - assert(appApi.classify('{"event": "progress"}') === "json", "进度 JSON 未归入 json"); - assert(appApi.highlight("abc", "b").includes("b"), "过滤高亮未生效"); - const marked = appApi.highlight("