revert: 移除全部机器学习代码，回退到无 ML 版本
删除 RL/HRL/训练前端及其文档、脚本、测试与依赖：
- src/mcm26b/{rl,hrl,training_ui}/、planner/{rl,hrl}_loop.py、process_control.py
- project/（PPO 策略、训练、环境、评估）
- scripts/{train_rl,train_hrl_*,benchmark_rl,benchmark_hrl,pretrain_*_expert 等}
- tests/{test_rl*,test_hrl*,test_training_ui*}、tests/js/training_ui_harness.cjs
- docs/{algo-rl,hrl-*,training-ui*} 与相关计划条目、论文 PDF 回退
CLI 恢复为 grid/seven/bayes + directional，移除 --strategy rl|hrl、
--q4-strategy、--rl-model、--hrl-* 等参数；pyproject 删除 rl 依赖组与
PyTorch CUDA index，uv.lock 同步重生成。
保留此后的非 ML 改动（referee 回放、Q1/Q3 脚本、代码清理等）。
验证：pytest 232 passed；Q3/Q4 --self-test 全清通过。
This commit is contained in:
Horldsence
+7
-62
@@ -30,7 +30,6 @@ from .protocol.log import SessionLog
|
||||
from .protocol.rules import EXIT_USER
|
||||
|
||||
Q3_STRATEGIES = {"grid": OmnidirectionalLoop, "seven": SevenBatch, "bayes": BayesLoop}
|
||||
Q4_STRATEGIES = {"directional": DirectionalExtension}
|
||||
|
||||
|
||||
def _load_referee():
|
||||
@@ -252,44 +251,13 @@ def _run(args) -> int:
|
||||
log = SessionLog(path=Path(args.log)) if args.log else SessionLog()
|
||||
client = SimulatorClient(args.robot_id, config=config, transport=transport, log=log)
|
||||
if args.problem == 4:
|
||||
strategy = args.q4_strategy
|
||||
loop_cls = Q4_STRATEGIES.get(strategy)
|
||||
strategy = "directional"
|
||||
loop_cls = DirectionalExtension
|
||||
else:
|
||||
strategy = args.strategy
|
||||
loop_cls = Q3_STRATEGIES.get(strategy)
|
||||
high_policy = None
|
||||
low_policy = None
|
||||
loop_cls = Q3_STRATEGIES[strategy]
|
||||
try:
|
||||
if strategy == "rl":
|
||||
# 惰性导入：默认运行闭环不加载 numpy/torch/stable-baselines3。
|
||||
from .planner.rl_loop import RLBayesLoop, RLDirectionalLoop
|
||||
loop_cls = RLBayesLoop if args.problem == 3 else RLDirectionalLoop
|
||||
elif strategy == "hrl":
|
||||
# P1：高层 MaskablePPO 调度。缺少模型时显式报错，不降级伪装。
|
||||
if not args.hrl_model:
|
||||
raise RuntimeError(
|
||||
"--strategy hrl/--q4-strategy hrl 需要 --hrl-model；"
|
||||
"无模型启发式请使用 benchmark_hrl.py 的 hrl_heuristic 变体。")
|
||||
from .hrl.policy import HighLevelPolicy, LowLevelPolicy
|
||||
from .planner.hrl_loop import HRLLoop
|
||||
high_policy = HighLevelPolicy(
|
||||
args.hrl_model, device=args.hrl_device, problem=args.problem)
|
||||
if args.hrl_low_model:
|
||||
low_policy = LowLevelPolicy(args.hrl_low_model, device=args.hrl_device)
|
||||
loop_cls = HRLLoop
|
||||
if loop_cls is None:
|
||||
raise RuntimeError(f"未知策略: {strategy}")
|
||||
if strategy == "rl":
|
||||
summary = loop_cls(client, config, model_path=args.rl_model).run()
|
||||
elif strategy == "hrl":
|
||||
summary = loop_cls(
|
||||
client, config, problem=args.problem, high_policy=high_policy,
|
||||
low_policy=low_policy, trace_executor=args.trace_low).run()
|
||||
else:
|
||||
summary = loop_cls(client, config).run()
|
||||
except RuntimeError as exc:
|
||||
print(f"策略不可用: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
summary = loop_cls(client, config).run()
|
||||
finally:
|
||||
if judge is not None:
|
||||
_settle_local_judge(judge)
|
||||
@@ -301,10 +269,7 @@ def _run(args) -> int:
|
||||
def _finalize_run(args, summary, strategy, *, truth, judge, judge_session,
|
||||
real_budget_s) -> int:
|
||||
"""补齐 summary 元信息、核对真值并输出；返回进程退出码。"""
|
||||
if args.problem == 4:
|
||||
summary["strategy"] = "rl_q4" if strategy == "rl" else strategy
|
||||
else:
|
||||
summary["strategy"] = strategy
|
||||
summary["strategy"] = strategy
|
||||
if real_budget_s is not None:
|
||||
summary["self_test_real_budget_s"] = real_budget_s
|
||||
if args.summary_steps is not None and len(summary.get("steps", [])) > args.summary_steps:
|
||||
@@ -349,30 +314,10 @@ def main(argv=None) -> int:
|
||||
run.add_argument("--log", default=None, help="会话日志 JSONL 输出路径")
|
||||
run.add_argument("--problem", type=int, choices=(3, 4), default=3,
|
||||
help="问题3=全向源，问题4=混合定向源（默认 3）")
|
||||
run.add_argument("--strategy", choices=tuple(Q3_STRATEGIES) + ("rl", "hrl"),
|
||||
default="bayes",
|
||||
run.add_argument("--strategy", choices=tuple(Q3_STRATEGIES), default="bayes",
|
||||
help="问题3策略：bayes=粒子滤波+信息增益+滚动调度（默认，"
|
||||
"自建消融相对 seven 再降 19.5%%），seven=七点批量，"
|
||||
"grid=49点网格基线，rl=PPO 强化学习选点（需先训练模型），"
|
||||
"hrl=高层 MaskablePPO 分层调度（需 --hrl-model）；"
|
||||
"仅 --problem 3 生效")
|
||||
run.add_argument("--q4-strategy", choices=tuple(Q4_STRATEGIES) + ("rl", "hrl"),
|
||||
default="directional",
|
||||
help="问题4策略：directional=49点覆盖+遮挡梯子+后备清除（默认），"
|
||||
"rl=PPO 强化学习选点（需先训练模型），"
|
||||
"hrl=高层 MaskablePPO 分层调度（需 --hrl-model）；仅 --problem 4 生效")
|
||||
run.add_argument("--rl-model", default=None,
|
||||
help="RL 模型路径（默认 output/rl/q3_ppo.zip 或 "
|
||||
"output/rl/q4_ppo.zip；训练脚本 scripts/train_rl.py）")
|
||||
run.add_argument("--hrl-model", default=None,
|
||||
help="高层 HRL MaskablePPO 模型路径（--strategy hrl 时必填；"
|
||||
"训练脚本 scripts/train_hrl_high.py）")
|
||||
run.add_argument("--hrl-low-model", default=None,
|
||||
help="可选低层 HRL 模型；当前只用于轨迹推演留痕，不改变协议终点")
|
||||
run.add_argument("--hrl-device", choices=("auto", "cpu", "cuda", "mps"),
|
||||
default="auto", help="HRL 模型推理设备")
|
||||
run.add_argument("--trace-low", action="store_true",
|
||||
help="开启低层到达推演留痕（本地虚拟轨迹，不增加协议耗时）")
|
||||
"grid=49点网格基线；仅 --problem 3 生效")
|
||||
run.add_argument("--bearing-margin-deg", type=float, default=0.005,
|
||||
help="示向度舍入裕量（服务器保留两位小数；默认 0.005°）")
|
||||
run.add_argument("--real-budget-s", type=float, default=None,
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
"""分层强化学习实验：整场子目标调度环境与独立的八方向到达实验。
|
||||
|
||||
与 :mod:`mcm26b.rl`（单步 PPO 选点）的差异：高层在整场 10–16 源的统一
|
||||
候选槽位上做调度决策（证书清除 / 信息增益定位 / 覆盖站扫频 / 后备清除），
|
||||
当前运行闭环将子目标端点提交协议，八方向策略仅用于可选本地路径留痕，
|
||||
不影响真实请求及虚拟计时。完整高层训练、CLI 与配对评估仍待接通；
|
||||
优化架构及逐项验收见 docs/hrl-sol-execution.md。
|
||||
|
||||
信念（belief + 网格幸存集）、证书审批与后备清除复用 planner 基础设施；
|
||||
每次改动仍须验证真源保留和真实全清，不能用内部状态自洽替代评测。
|
||||
|
||||
本包属于训练/推理侧组件，运行时对依赖保持惰性（stable-baselines3 /
|
||||
sb3-contrib 仅在显式加载模型时导入）。
|
||||
"""
|
||||
@@ -1,49 +0,0 @@
|
||||
"""HRL 观测/动作空间常量：模型接口的唯一字面量来源。
|
||||
|
||||
槽位布局（训练与推理共享，改动任一值都会使已训练模型失配，须同步重训）：
|
||||
|
||||
0–19 clear 频道 c 的证书清除（槽位 = c-1，证书就绪才有效）
|
||||
20–39 locate 频道 c 的定位测量（信息增益选点或定向遮挡梯子探点）
|
||||
40–88 station 覆盖站（问题3：七点环 0–6 有效；问题4：49 点网格）
|
||||
89–108 fallback 频道 c 的后备网格清除（仅问题4，遮挡梯子耗尽后开放）
|
||||
109–119 保留（恒屏蔽）
|
||||
|
||||
高层观测 = 全局 8 维 + 20 频道 × 14 维 + 120 槽位 × 8 维 = 1248 维；
|
||||
低层观测 10 维、动作 Discrete(8)（8 方向，步长 ≤100 m）。
|
||||
"""
|
||||
|
||||
FEATURE_VERSION = 4
|
||||
ACTION_LAYOUT_VERSION = 1
|
||||
REWARD_VERSION = 1
|
||||
CHANNEL_COUNT = 20
|
||||
|
||||
# ------------------------------------------------------------ 高层动作槽位
|
||||
CLEAR_SLOTS = CHANNEL_COUNT
|
||||
LOCATE_SLOTS = CHANNEL_COUNT
|
||||
STATION_SLOTS = 49
|
||||
FALLBACK_SLOTS = CHANNEL_COUNT
|
||||
CLEAR_OFFSET = 0
|
||||
LOCATE_OFFSET = CLEAR_OFFSET + CLEAR_SLOTS
|
||||
STATION_OFFSET = LOCATE_OFFSET + LOCATE_SLOTS
|
||||
FALLBACK_OFFSET = STATION_OFFSET + STATION_SLOTS
|
||||
N_CANDIDATES = 120
|
||||
|
||||
# ------------------------------------------------------------ 高层观测
|
||||
PER_CHANNEL_DIM = 14
|
||||
HIGH_GLOBAL_DIM = 8
|
||||
PER_CANDIDATE_DIM = 8
|
||||
HIGH_OBS_DIM = (HIGH_GLOBAL_DIM + CHANNEL_COUNT * PER_CHANNEL_DIM
|
||||
+ N_CANDIDATES * PER_CANDIDATE_DIM) # 1248
|
||||
|
||||
# ------------------------------------------------------------ 低层接口
|
||||
LOW_OBS_DIM = 10
|
||||
LOW_NUM_ACTIONS = 8 # 8 方向，45° 等分
|
||||
LOW_STEP_M = 100.0 # 每步最大步长 [题面 低层动作 50–100 m]
|
||||
LOW_REACH_M = 20.0 # 到达阈值（与清除半径一致）
|
||||
LOW_MAX_STEPS = 60 # 直径穿越 ~51 步上界
|
||||
|
||||
# ------------------------------------------------------------ 观测归一化
|
||||
POS_SCALE_M = 2000.0 # 位置/位移归一化尺度（Ω 半径 1800 m 外扩）
|
||||
RADIUS_SCALE_M = 2000.0
|
||||
COST_SCALE_S = 600.0 # 槽位预计服务耗时归一化尺度
|
||||
MEASURE_COUNT_SCALE = 12.0
|
||||
@@ -1,109 +0,0 @@
|
||||
"""课程学习与域随机化：训练场景按阶段由易到难（prompt §7）。
|
||||
|
||||
四阶段课程：1 源/5 频道/无噪声 → 3 源/10 频道/±0.5° → 8 源/16 频道/±1°
|
||||
→ 10–16 源/20 频道/±1°（问题4 最后阶段按比例混入定向源）。域随机化
|
||||
覆盖源位置（Ω 圆域均匀）、频道（互异抽样）、有效接收半径、朝向与逐局
|
||||
误差种子 [prompt §7.2]；机器狗初始位置固定 (0,0)——协议语义如此，
|
||||
随机化它反而制造训练-部署差异（与 v1"训练部署特征错位"教训一致）。
|
||||
|
||||
真值只进入训练/评估场景生成，不进入任何观测；策略端仅见模拟器响应。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from random import Random
|
||||
|
||||
from ..config import ProblemConfig
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Stage:
|
||||
"""课程阶段参数：源数范围、频道上限、示向度噪声与定向源比例。"""
|
||||
|
||||
name: str
|
||||
sources_min: int
|
||||
sources_max: int
|
||||
channel_max: int # 频道 > channel_max 的频道保证无源
|
||||
bearing_error_half_deg: float
|
||||
directional_ratio: float # 问题4 最后阶段定向源比例；问题3 恒 0
|
||||
recommended_timesteps: int
|
||||
|
||||
|
||||
STAGES: tuple[Stage, ...] = (
|
||||
Stage("s1-single", 1, 1, 5, 0.0, 0.0, 60_000),
|
||||
Stage("s2-few", 3, 3, 10, 0.5, 0.0, 80_000),
|
||||
Stage("s3-many", 8, 8, 16, 1.0, 0.0, 100_000),
|
||||
Stage("s4-full", 10, 16, 20, 1.0, 0.4, 140_000),
|
||||
)
|
||||
|
||||
|
||||
def stage_with_noise(config: ProblemConfig, stage: Stage) -> ProblemConfig:
|
||||
"""把阶段噪声写入 ProblemConfig（belief/PF 的保守裕量随之同步）。"""
|
||||
field = dataclasses.replace(
|
||||
config.field, bearing_error_half_deg=stage.bearing_error_half_deg)
|
||||
return dataclasses.replace(config, field=field)
|
||||
|
||||
|
||||
# 快速模式只减少选点评分的样本，不改变证明使用的幸存网格。29.3m 格距的
|
||||
# Voronoi 裕量约20.72m，已经超过20m清除半径，会使粒子证书永远不可达。
|
||||
# 默认使用生产配置；快速模式必须显式指定并在训练元数据记录。
|
||||
TRAINING_SCORE_SUBSAMPLE = 240
|
||||
TRAINING_MC_REPRESENTATIVES = 5
|
||||
|
||||
|
||||
def training_config(config: ProblemConfig, stage: Stage, *,
|
||||
fidelity: str = "production") -> ProblemConfig:
|
||||
"""阶段噪声配置；fast只降低评分采样量，production与部署同参数。"""
|
||||
if fidelity not in ("production", "fast"):
|
||||
raise ValueError("fidelity 必须是 production 或 fast")
|
||||
noisy = stage_with_noise(config, stage)
|
||||
if fidelity == "production":
|
||||
return noisy
|
||||
bayes = dataclasses.replace(
|
||||
noisy.bayes,
|
||||
score_subsample=TRAINING_SCORE_SUBSAMPLE,
|
||||
mc_representatives=TRAINING_MC_REPRESENTATIVES)
|
||||
return dataclasses.replace(noisy, bayes=bayes)
|
||||
|
||||
|
||||
def make_scene(rng: Random, config: ProblemConfig, problem: int,
|
||||
stage: Stage, *, domain_randomization: bool = True
|
||||
) -> tuple[list, list[int]]:
|
||||
"""生成一局课程场景，返回 (referee.Source 列表, 已占用频道列表)。
|
||||
|
||||
源位置在 Ω 圆域均匀分布、频道从 1..stage.channel_max 互异抽样、
|
||||
有效接收半径 [1000,1500] 均匀 [题面]；问题4 按 stage.directional_ratio
|
||||
混入定向源，朝向均匀。domain_randomization=False 时只用 rng 的确定
|
||||
序列（仍随机但可整场复现），供评估固定场景对照。
|
||||
"""
|
||||
from referee import Source
|
||||
|
||||
field = config.field
|
||||
count = rng.randint(stage.sources_min, stage.sources_max)
|
||||
channels = rng.sample(range(field.channel_min, stage.channel_max + 1), count)
|
||||
sources = []
|
||||
for channel in channels:
|
||||
radius = field.omega_radius_m * math.sqrt(rng.random())
|
||||
angle = rng.uniform(0.0, 2.0 * math.pi)
|
||||
r_eff = rng.uniform(field.r_eff_min_m, field.r_eff_max_m)
|
||||
directional = (problem == 4
|
||||
and rng.random() < stage.directional_ratio)
|
||||
heading = rng.uniform(0.0, 360.0) if directional else 0.0
|
||||
if not domain_randomization:
|
||||
# 固定化退耦：半径/朝向取区间代表值，保留位置随机（评估用）。
|
||||
r_eff = field.r_eff_min_m
|
||||
sources.append(Source(channel=channel, x=radius * math.cos(angle),
|
||||
y=radius * math.sin(angle), r_eff_m=r_eff,
|
||||
directional=directional, heading_deg=heading))
|
||||
return sources, channels
|
||||
|
||||
|
||||
def training_spec(config: ProblemConfig, stage: Stage, *,
|
||||
max_real_duration_s: float = 3600.0):
|
||||
"""带阶段噪声的 referee Spec（训练侧使用；正式对接以 /enter 为准）。"""
|
||||
from ..harness import referee_spec
|
||||
return referee_spec(stage_with_noise(config, stage),
|
||||
max_real_duration_s=max_real_duration_s)
|
||||
@@ -1,113 +0,0 @@
|
||||
"""低层到达执行器与观测：8 方向步进（prompt §4.2），训练/部署同源。
|
||||
|
||||
低层只负责"怎么去"：给定当前位置与子目标，输出离散 8 方向步进序列。
|
||||
步长每步 ≤ ``LOW_STEP_M``，近目标时缩短步长；到达阈值 ``LOW_REACH_M``
|
||||
与清除半径一致。推演完全本地，不与模拟器交互、不产生虚拟耗时——协议
|
||||
语义下移动耗时 = 两次动作位置间的直线距离 ÷ 速度，仅由终点决定。
|
||||
|
||||
本模块仅供独立导航实验和轨迹诊断；当前 HTTP 执行路径仍直接服务高层
|
||||
目标，不能据此声称低层 PPO 改善正式任务耗时。policy=None 使用解析
|
||||
直线控制器；PPO 推演超时不会免费跳到目标。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
import numpy as np
|
||||
|
||||
from ..config import Vec2
|
||||
from .constants import (LOW_MAX_STEPS, LOW_NUM_ACTIONS,
|
||||
LOW_REACH_M, LOW_STEP_M, POS_SCALE_M)
|
||||
|
||||
__all__ = ["build_low_observation", "LowLevelExecutor", "action_bearing",
|
||||
"navigation_transition"]
|
||||
|
||||
|
||||
def action_bearing(action: int) -> float:
|
||||
"""离散方向 -> 弧度方位（0 = +x，逆时针 45° 等分）。"""
|
||||
return float(action) * 2.0 * math.pi / LOW_NUM_ACTIONS
|
||||
|
||||
|
||||
def navigation_transition(position: Vec2, goal: Vec2, action: int, *,
|
||||
step_m: float = LOW_STEP_M) -> tuple[Vec2, float]:
|
||||
"""训练与轨迹推演共享的运动学；返回位置和实际位移长度。
|
||||
|
||||
方向仍由策略决定，近目标时仅缩短步长，不把位置吸附到目标。
|
||||
这是无障碍平面导航实验；不引入观测不可见的绝对位置裁剪。
|
||||
"""
|
||||
if not 0 <= int(action) < LOW_NUM_ACTIONS:
|
||||
raise ValueError("低层动作必须是 0..7")
|
||||
if not math.isfinite(step_m) or step_m <= 0:
|
||||
raise ValueError("step_m 必须是有限正数")
|
||||
distance = min(float(step_m), math.dist(position, goal))
|
||||
bearing = action_bearing(int(action))
|
||||
next_position = (position[0] + distance * math.cos(bearing),
|
||||
position[1] + distance * math.sin(bearing))
|
||||
return next_position, math.dist(position, next_position)
|
||||
|
||||
|
||||
def build_low_observation(position: Vec2, goal: Vec2, prev_action: int | None,
|
||||
steps: int) -> np.ndarray:
|
||||
"""低层观测（shape=(LOW_OBS_DIM,)，训练/推理逐位一致）。"""
|
||||
dx, dy = goal[0] - position[0], goal[1] - position[1]
|
||||
distance = math.hypot(dx, dy)
|
||||
bearing = math.atan2(dy, dx)
|
||||
prev_sin = prev_cos = 0.0
|
||||
if prev_action is not None:
|
||||
prev_bearing = action_bearing(prev_action)
|
||||
prev_sin, prev_cos = math.sin(prev_bearing), math.cos(prev_bearing)
|
||||
return np.asarray([
|
||||
dx / POS_SCALE_M,
|
||||
dy / POS_SCALE_M,
|
||||
distance / POS_SCALE_M,
|
||||
math.sin(bearing),
|
||||
math.cos(bearing),
|
||||
steps / LOW_MAX_STEPS,
|
||||
prev_sin,
|
||||
prev_cos,
|
||||
min(distance / LOW_STEP_M, 3.0) / 3.0,
|
||||
1.0,
|
||||
], dtype=np.float32)
|
||||
|
||||
|
||||
class LowLevelExecutor:
|
||||
"""子目标到达推演：低层 PPO 或直线控制器（policy=None）。"""
|
||||
|
||||
def __init__(self, policy=None, *, step_m: float = LOW_STEP_M,
|
||||
reach_m: float = LOW_REACH_M, max_steps: int = LOW_MAX_STEPS):
|
||||
self.policy = policy
|
||||
self.step_m = float(step_m)
|
||||
self.reach_m = float(reach_m)
|
||||
self.max_steps = int(max_steps)
|
||||
self.last_roll_info: dict = {}
|
||||
|
||||
def roll(self, start: Vec2, goal: Vec2) -> list[Vec2]:
|
||||
"""返回真实推演途经点；last_roll_info 如实记录到达/截断。"""
|
||||
goal = (float(goal[0]), float(goal[1]))
|
||||
if self.policy is None:
|
||||
self.last_roll_info = {"reached": True, "truncated": False,
|
||||
"final_distance_m": 0.0,
|
||||
"path_length_m": math.dist(start, goal)}
|
||||
return [goal]
|
||||
position = (float(start[0]), float(start[1]))
|
||||
waypoints: list[Vec2] = []
|
||||
prev_action: int | None = None
|
||||
path_length = 0.0
|
||||
for steps in range(self.max_steps):
|
||||
if math.dist(position, goal) <= self.reach_m:
|
||||
break
|
||||
observation = build_low_observation(position, goal, prev_action,
|
||||
steps)
|
||||
action = int(self.policy.select(observation))
|
||||
position, moved = navigation_transition(position, goal, action,
|
||||
step_m=self.step_m)
|
||||
path_length += moved
|
||||
waypoints.append(position)
|
||||
prev_action = action
|
||||
distance = math.dist(position, goal)
|
||||
self.last_roll_info = {"reached": distance <= self.reach_m,
|
||||
"truncated": distance > self.reach_m,
|
||||
"final_distance_m": distance,
|
||||
"path_length_m": path_length}
|
||||
return waypoints
|
||||
@@ -1,260 +0,0 @@
|
||||
"""高层 PPO 训练环境：整场 10–16 源的发现-定位-清除调度（prompt §2/§5.2）。
|
||||
|
||||
与 v2 :class:`~mcm26b.rl.env.LocalizationEnv`（单源 ≤16 步子任务）的本质
|
||||
差异：本环境的每一步是一次完整的高层调度决策（候选槽位选择 + 任务执行），
|
||||
环境语义与运行闭环 :class:`~mcm26b.planner.hrl_loop.HRLLoop` 共用同一
|
||||
混入层（hrl.scheduler），训练-部署观测逐位同源。
|
||||
|
||||
奖励（prompt §5.2 的可折算实现，权重见 :class:`HRLReward`）：
|
||||
r = -Δ虚拟耗时/20 + 100·清除 + 5·新发现 + 0.5·Σ熵减 - 2·冗余检测
|
||||
终止：全部频道清除/证明缺席（+50）；截断仍有未决频道（-100）；异常（-100）。
|
||||
主口径是虚拟耗时——与题目目标函数（最小化平均定位清除时间/虚拟总时间）
|
||||
一致，清除/发现/熵减是塑形项（消融实验逐项验证其必要性）。
|
||||
|
||||
真值只在 reset 布景时使用，绝不进入观测；感知读数全部来自
|
||||
referee.LocalSimulator（与官方 HTTP 模拟器同协议语义）。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from random import Random
|
||||
|
||||
import gymnasium as gym
|
||||
import numpy as np
|
||||
from gymnasium import spaces
|
||||
|
||||
from ..belief import ChannelStatus
|
||||
from ..config import ProblemConfig
|
||||
from ..hrl.curriculum import STAGES, Stage, make_scene, training_config
|
||||
from ..hrl.constants import HIGH_OBS_DIM, N_CANDIDATES
|
||||
from ..planner.hrl_loop import HRLLoop
|
||||
from ..protocol.client import SimulatorClient
|
||||
from ..protocol.log import SessionLog
|
||||
|
||||
__all__ = ["HRLReward", "HierarchicalEnv"]
|
||||
|
||||
|
||||
@dataclass
|
||||
class HRLReward:
|
||||
"""高层奖励权重。
|
||||
|
||||
默认保留 prompt §5.2 的可解释塑形，便于对照实验；P0.4 建议的
|
||||
纯时间口径可通过 :meth:`time_only` 显式启用，训练脚本默认使用它。
|
||||
"""
|
||||
|
||||
time_scale: float = 20.0 # r -= Δ虚拟耗时 / time_scale
|
||||
clear: float = 100.0 # 每清除一个源
|
||||
detect: float = 5.0 # 每新发现一个频道有信号
|
||||
info_weight: float = 0.5 # 粒子云熵减（log 幸存格数）内在奖励
|
||||
redundant: float = 2.0 # 同频道同点重复检测惩罚
|
||||
invalid: float = 2.0 # 掩码外动作（无掩码消融路径）
|
||||
terminal_accounted: float = 50.0 # 全部频道清除/证毕的收尾奖励
|
||||
terminal_unresolved: float = 100.0 # 截断/停机时仍有未决频道的惩罚
|
||||
failure: float = 100.0 # 协议/守恒性异常惩罚
|
||||
|
||||
@classmethod
|
||||
def time_only(cls) -> "HRLReward":
|
||||
"""P0.4 纯时间目标：只惩罚已发生虚拟时间，去掉塑形项。
|
||||
|
||||
终止奖/罚是固定常数，不逐源叠加；失败罚按虚拟时限/时间尺度
|
||||
设定为足以覆盖一整局时间成本。
|
||||
"""
|
||||
return cls(
|
||||
time_scale=100.0,
|
||||
clear=0.0,
|
||||
detect=0.0,
|
||||
info_weight=0.0,
|
||||
redundant=0.0,
|
||||
invalid=2.0,
|
||||
terminal_accounted=30.0,
|
||||
terminal_unresolved=300.0,
|
||||
failure=3600.0,
|
||||
)
|
||||
|
||||
|
||||
class HierarchicalEnv(gym.Env):
|
||||
"""整场分层调度 Gymnasium 环境（问题3/4，课程参数化）。
|
||||
|
||||
stage=None 时使用 STAGES 末阶段（完整任务口径）；课程训练按阶段
|
||||
传入。domain_randomization=False 保留随机位置但固定半径代表值，
|
||||
供配对评估。
|
||||
"""
|
||||
|
||||
metadata = {"render_modes": []}
|
||||
|
||||
def __init__(self, config: ProblemConfig | None = None, *,
|
||||
problem: int = 3, stage: Stage | None = None,
|
||||
max_episode_steps: int = 300,
|
||||
domain_randomization: bool = True,
|
||||
reward: HRLReward | None = None,
|
||||
real_budget_s: float = 3600.0,
|
||||
error_seed: int | None = None,
|
||||
fidelity: str = "production"):
|
||||
super().__init__()
|
||||
if problem not in (3, 4):
|
||||
raise ValueError("problem 必须是 3 或 4")
|
||||
self.base_config = config or ProblemConfig()
|
||||
self.problem = int(problem)
|
||||
self.stage = stage or STAGES[-1]
|
||||
self.max_episode_steps = int(max_episode_steps)
|
||||
self.domain_randomization = bool(domain_randomization)
|
||||
self.reward = reward or HRLReward()
|
||||
self.real_budget_s = float(real_budget_s)
|
||||
self.error_seed = error_seed
|
||||
self.fidelity = fidelity
|
||||
# 默认与部署同精度；fast仅减少评分采样，保留证书网格精度。
|
||||
self.train_config = training_config(self.base_config, self.stage,
|
||||
fidelity=fidelity)
|
||||
self.observation_space = spaces.Box(low=-12.0, high=12.0,
|
||||
shape=(HIGH_OBS_DIM,),
|
||||
dtype=np.float32)
|
||||
self.action_space = spaces.Discrete(N_CANDIDATES)
|
||||
|
||||
self._scene_rng = Random()
|
||||
self._loop: HRLLoop | None = None
|
||||
self._client: SimulatorClient | None = None
|
||||
self._steps = 0
|
||||
self._terminated = False
|
||||
self._truncated = False
|
||||
self._start_virtual_s = 0.0
|
||||
self._current_candidates: list = []
|
||||
self._failure_reason: str | None = None
|
||||
self._last_outcome = None
|
||||
|
||||
# ------------------------------------------------------------ 场景
|
||||
def _make_episode(self) -> None:
|
||||
from ..harness import LocalSimTransport
|
||||
from referee import LocalSimulator
|
||||
|
||||
sources, _channels = make_scene(
|
||||
self._scene_rng, self.train_config, self.problem, self.stage,
|
||||
domain_randomization=self.domain_randomization)
|
||||
self._simulator = LocalSimulator(
|
||||
sources, robot_id="hrl-train",
|
||||
spec=self._training_spec(),
|
||||
error_seed=(self.error_seed if self.error_seed is not None
|
||||
else self._scene_rng.randint(0, 2 ** 31 - 1)),
|
||||
stress_mode=True)
|
||||
self._client = SimulatorClient(
|
||||
"hrl-train", config=self.train_config,
|
||||
transport=LocalSimTransport(self._simulator), log=SessionLog())
|
||||
self._client.enter()
|
||||
self._loop = HRLLoop(self._client, self.train_config,
|
||||
problem=self.problem)
|
||||
|
||||
def _training_spec(self):
|
||||
from ..hrl.curriculum import training_spec
|
||||
return training_spec(self.train_config, self.stage,
|
||||
max_real_duration_s=self.real_budget_s)
|
||||
|
||||
# ------------------------------------------------------------ Gym API
|
||||
def reset(self, *, seed: int | None = None, options: dict | None = None):
|
||||
super().reset(seed=seed)
|
||||
if seed is not None:
|
||||
self._scene_rng = Random(seed)
|
||||
self._steps = 0
|
||||
self._terminated = False
|
||||
self._truncated = False
|
||||
self._failure_reason = None
|
||||
self._make_episode()
|
||||
self._start_virtual_s = self._client.state.virtual_time_s
|
||||
self._current_candidates = self._loop.build_candidates()
|
||||
observation = self._loop.hrl_observation(self._current_candidates)
|
||||
return observation, self._info()
|
||||
|
||||
def action_masks(self) -> np.ndarray:
|
||||
"""MaskablePPO 动作掩码接口（与观测同源的候选槽位有效性）。"""
|
||||
return self._loop.hrl_action_mask(self._current_candidates)
|
||||
|
||||
def _info(self) -> dict:
|
||||
loop = self._loop
|
||||
belief = loop.belief
|
||||
cleared = len(loop.cleared)
|
||||
absent = sum(1 for ch in belief.channels.values()
|
||||
if ch.status is ChannelStatus.ABSENT)
|
||||
return {
|
||||
"steps": self._steps,
|
||||
"cleared": cleared,
|
||||
"absent_proven": absent,
|
||||
"accounted": cleared + absent,
|
||||
"virtual_time_s": self._client.state.virtual_time_s,
|
||||
"episode_virtual_cost_s": (self._client.state.virtual_time_s
|
||||
- self._start_virtual_s),
|
||||
"stop_reason": loop.stop_reason,
|
||||
"failure_reason": self._failure_reason,
|
||||
"anomalies": list(loop.anomalies),
|
||||
}
|
||||
|
||||
def step(self, action):
|
||||
if self._terminated or self._truncated:
|
||||
raise RuntimeError("step() 需要在 terminated/truncated 后先 reset()")
|
||||
loop = self._loop
|
||||
self._steps += 1
|
||||
try:
|
||||
outcome = loop.hrl_step(action, candidates=self._current_candidates)
|
||||
except Exception as exc: # 协议/守恒性异常：与 v2 环境同口径收尾
|
||||
self._failure_reason = type(exc).__name__ + ': ' + str(exc)
|
||||
observation = np.zeros((HIGH_OBS_DIM,), dtype=np.float32)
|
||||
self._terminated = True
|
||||
return (observation, -self.reward.failure, True, False,
|
||||
self._info())
|
||||
self._last_outcome = outcome
|
||||
|
||||
if outcome.invalid:
|
||||
reward = -self.reward.invalid
|
||||
else:
|
||||
reward = self._reward_from(outcome)
|
||||
|
||||
# 终止判定：停机原因/全部证毕；或候选耗尽时走运行闭环 run() 的
|
||||
# 同款安全网（覆盖缺席标记 + 阻塞定位清除），安全网的虚拟耗时
|
||||
# 计入本步奖励（归因给促成该局面的决策序列）。
|
||||
terminated = loop.stop_reason is not None or loop._all_accounted()
|
||||
if not terminated:
|
||||
self._current_candidates = loop.build_candidates()
|
||||
if not self._current_candidates:
|
||||
virtual_before = self._client.state.virtual_time_s
|
||||
loop._mark_absent_after_coverage()
|
||||
loop._resolve_detected()
|
||||
reward -= (self._client.state.virtual_time_s - virtual_before) \
|
||||
/ self.reward.time_scale
|
||||
self._current_candidates = []
|
||||
terminated = True
|
||||
else:
|
||||
self._current_candidates = []
|
||||
|
||||
if terminated:
|
||||
reward += (self.reward.terminal_accounted
|
||||
if loop._all_accounted()
|
||||
else -self.reward.terminal_unresolved)
|
||||
truncated = False
|
||||
if not terminated and self._steps >= self.max_episode_steps:
|
||||
truncated = True
|
||||
if not loop._all_accounted():
|
||||
reward -= self.reward.terminal_unresolved
|
||||
self._terminated = terminated
|
||||
self._truncated = truncated
|
||||
if terminated or truncated:
|
||||
observation = np.zeros((HIGH_OBS_DIM,), dtype=np.float32)
|
||||
else:
|
||||
self._current_candidates = loop.build_candidates()
|
||||
observation = loop.hrl_observation(self._current_candidates)
|
||||
return observation, float(reward), terminated, truncated, self._info()
|
||||
|
||||
def _reward_from(self, outcome) -> float:
|
||||
reward = -outcome.delta_virtual_s / self.reward.time_scale
|
||||
reward += self.reward.clear * len(outcome.clear_success)
|
||||
reward += self.reward.detect * len(outcome.new_detections)
|
||||
reward += self.reward.info_weight * sum(outcome.info_gains.values())
|
||||
if outcome.redundant:
|
||||
reward -= self.reward.redundant
|
||||
return reward
|
||||
|
||||
def close(self):
|
||||
client = getattr(self, "_client", None)
|
||||
if client is not None and client.state.entered \
|
||||
and not client.state.exited:
|
||||
try:
|
||||
client.exit()
|
||||
except Exception:
|
||||
pass
|
||||
@@ -1,158 +0,0 @@
|
||||
"""低层 PPO 训练环境：随机起点-子目标的 8 方向到达任务（prompt §4.2/§5.1）。
|
||||
|
||||
奖励为实际位移耗时惩罚、转向惩罚与到达奖励。步长上限 100m，近目标
|
||||
缩短步长，避免固定步长振荡。此环境仅用于独立导航实验；正式 HTTP
|
||||
路径尚不执行低层 PPO 的途经点，不能将导航成绩当成正式任务收益。
|
||||
|
||||
策略端只见相对位姿观测，真值（子目标坐标）经相对向量进入观测，
|
||||
与部署执行器完全同源（build_low_observation）。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from random import Random
|
||||
|
||||
import gymnasium as gym
|
||||
import numpy as np
|
||||
from gymnasium import spaces
|
||||
|
||||
from ..config import ProblemConfig
|
||||
from .constants import (LOW_MAX_STEPS, LOW_NUM_ACTIONS, LOW_OBS_DIM,
|
||||
LOW_REACH_M, LOW_STEP_M)
|
||||
from .executor import (action_bearing, build_low_observation,
|
||||
navigation_transition)
|
||||
|
||||
__all__ = ["LowLevelEnv"]
|
||||
|
||||
|
||||
class LowLevelEnv(gym.Env):
|
||||
"""低层到达任务：观测 10 维，动作 Discrete(8)，逐步计费。"""
|
||||
|
||||
metadata = {"render_modes": []}
|
||||
|
||||
def __init__(self, config: ProblemConfig | None = None, *,
|
||||
seed: int | None = None,
|
||||
step_m: float = LOW_STEP_M, reach_m: float = LOW_REACH_M,
|
||||
max_steps: int = LOW_MAX_STEPS,
|
||||
min_distance_m: float = 300.0,
|
||||
turn_penalty: float = 0.05, reach_bonus: float = 2.0,
|
||||
step_cost_scale: float = 100.0,
|
||||
truncation_penalty: float = 2.0):
|
||||
super().__init__()
|
||||
self.config = config or ProblemConfig()
|
||||
self.step_m = float(step_m)
|
||||
self.reach_m = float(reach_m)
|
||||
self.max_steps = int(max_steps)
|
||||
self.min_distance_m = float(min_distance_m)
|
||||
self.turn_penalty = float(turn_penalty)
|
||||
self.reach_bonus = float(reach_bonus)
|
||||
self.step_cost_scale = float(step_cost_scale)
|
||||
self.truncation_penalty = float(truncation_penalty)
|
||||
self.observation_space = spaces.Box(low=-np.inf, high=np.inf,
|
||||
shape=(LOW_OBS_DIM,),
|
||||
dtype=np.float32)
|
||||
self.action_space = spaces.Discrete(LOW_NUM_ACTIONS)
|
||||
self._rng = Random(seed)
|
||||
self._position = (0.0, 0.0)
|
||||
self._goal = (0.0, 0.0)
|
||||
self._steps = 0
|
||||
self._prev_action: int | None = None
|
||||
self._terminated = False
|
||||
self._truncated = False
|
||||
self._path_length_m = 0.0
|
||||
self._virtual_time_s = 0.0
|
||||
self._straight_m = 0.0
|
||||
|
||||
# ------------------------------------------------------------ Gym API
|
||||
def reset(self, *, seed: int | None = None, options: dict | None = None):
|
||||
super().reset(seed=seed)
|
||||
if seed is not None:
|
||||
self._rng = Random(seed)
|
||||
radius = self.config.field.omega_radius_m
|
||||
self._position = self._sample_point(radius)
|
||||
for _ in range(100):
|
||||
self._goal = self._sample_point(radius)
|
||||
if math.dist(self._position, self._goal) >= self.min_distance_m:
|
||||
break
|
||||
self._steps = 0
|
||||
self._prev_action = None
|
||||
self._terminated = False
|
||||
self._truncated = False
|
||||
self._path_length_m = 0.0
|
||||
self._virtual_time_s = 0.0
|
||||
self._straight_m = math.dist(self._position, self._goal)
|
||||
return self._observe(), self._info()
|
||||
|
||||
def action_masks(self) -> np.ndarray:
|
||||
"""无障碍低层八个方向均可用，兼容 MaskablePPO。"""
|
||||
return np.ones(LOW_NUM_ACTIONS, dtype=bool)
|
||||
|
||||
def _sample_point(self, radius: float) -> tuple[float, float]:
|
||||
angle = self._rng.uniform(0.0, 2.0 * math.pi)
|
||||
distance = radius * math.sqrt(self._rng.random())
|
||||
return (distance * math.cos(angle), distance * math.sin(angle))
|
||||
|
||||
def step(self, action):
|
||||
if self._terminated or self._truncated:
|
||||
raise RuntimeError("step() 需要在 terminated/truncated 后先 reset()")
|
||||
action = int(action)
|
||||
if self._prev_action is not None:
|
||||
turn = abs((action_bearing(action)
|
||||
- action_bearing(self._prev_action) + math.pi)
|
||||
% (2.0 * math.pi) - math.pi) / math.pi
|
||||
else:
|
||||
turn = 0.0
|
||||
self._position, moved = navigation_transition(
|
||||
self._position, self._goal, action, step_m=self.step_m)
|
||||
self._prev_action = action
|
||||
self._steps += 1
|
||||
|
||||
step_cost_s = moved / self.config.cost.speed_mps
|
||||
self._path_length_m += moved
|
||||
self._virtual_time_s += step_cost_s
|
||||
reward = -step_cost_s / self.step_cost_scale
|
||||
reward -= self.turn_penalty * turn
|
||||
distance = math.dist(self._position, self._goal)
|
||||
terminated = distance <= self.reach_m
|
||||
truncated = False
|
||||
if terminated:
|
||||
reward += self.reach_bonus
|
||||
elif self._steps >= self.max_steps:
|
||||
truncated = True
|
||||
reward -= self.truncation_penalty
|
||||
self._terminated = terminated
|
||||
self._truncated = truncated
|
||||
info = self._info()
|
||||
info["final_distance_m"] = distance
|
||||
info["step_distance_m"] = moved
|
||||
info["step_virtual_time_s"] = step_cost_s
|
||||
return self._observe(), float(reward), terminated, truncated, info
|
||||
|
||||
def _observe(self) -> np.ndarray:
|
||||
return build_low_observation(self._position, self._goal,
|
||||
self._prev_action, self._steps)
|
||||
|
||||
def _info(self) -> dict:
|
||||
return {
|
||||
"position": self._position,
|
||||
"goal": self._goal,
|
||||
"steps": self._steps,
|
||||
"distance_m": math.dist(self._position, self._goal),
|
||||
"straight_m": self._straight_m,
|
||||
"path_length_m": self._path_length_m,
|
||||
"virtual_time_s": self._virtual_time_s,
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------ 专家（BC）
|
||||
def expert_action(self) -> int:
|
||||
"""直线专家：选择方位最接近子目标方向的离散方向（BC 标签）。"""
|
||||
bearing = math.atan2(self._goal[1] - self._position[1],
|
||||
self._goal[0] - self._position[0])
|
||||
best, best_gap = 0, float("inf")
|
||||
for action in range(LOW_NUM_ACTIONS):
|
||||
gap = abs((action_bearing(action) - bearing + math.pi)
|
||||
% (2.0 * math.pi) - math.pi)
|
||||
if gap < best_gap:
|
||||
best, best_gap = action, gap
|
||||
return best
|
||||
@@ -1,135 +0,0 @@
|
||||
"""HRL 策略加载与推理（运行时接入 planner 的唯一入口，模式同 rl.policy）。
|
||||
|
||||
模型文件由 scripts/train_hrl_low.py / train_hrl_high.py 生成，并伴随同名
|
||||
JSON 元数据（feature_version/algorithm/problem/维度）；加载时校验，失配
|
||||
即拒绝运行，杜绝静默错位。stable-baselines3 / torch 均为惰性导入。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
from .constants import (ACTION_LAYOUT_VERSION, FEATURE_VERSION,
|
||||
HIGH_OBS_DIM, LOW_NUM_ACTIONS, LOW_OBS_DIM,
|
||||
N_CANDIDATES, REWARD_VERSION)
|
||||
|
||||
__all__ = ["HighLevelPolicy", "LowLevelPolicy",
|
||||
"default_high_model_path", "default_low_model_path"]
|
||||
|
||||
|
||||
def _resolve_device(device: str) -> str:
|
||||
if device != "auto":
|
||||
return device
|
||||
try:
|
||||
import torch
|
||||
return "cuda" if torch.cuda.is_available() else "cpu"
|
||||
except ImportError:
|
||||
return "cpu"
|
||||
|
||||
|
||||
def _load_maskable(model_path: Path, device: str, *,
|
||||
obs_dim: int, act_dim: int, problem: int):
|
||||
if not model_path.is_file():
|
||||
raise RuntimeError(
|
||||
f"未找到 HRL 模型 {model_path}。请先训练：\n"
|
||||
f" uv sync --group rl --group dev\n"
|
||||
f" uv run python scripts/train_hrl_high.py --problem {problem}")
|
||||
try:
|
||||
from sb3_contrib import MaskablePPO
|
||||
except ImportError as exc:
|
||||
raise RuntimeError(
|
||||
"缺少 RL 运行依赖（sb3-contrib）。请执行 `uv sync --group rl` "
|
||||
"后重试。") from exc
|
||||
meta_path = model_path.with_suffix('.json')
|
||||
metadata = json.loads(meta_path.read_text(encoding='utf-8')) \
|
||||
if meta_path.exists() else {}
|
||||
# 高层模型必须携带完整语义元数据；缺字段不能默认兼容，防止旧槽位/
|
||||
# 旧奖励语义模型被静默加载。低层导航模型只校验基础特征版本。
|
||||
strict = act_dim == N_CANDIDATES
|
||||
if strict:
|
||||
required = ("feature_version", "action_layout_version",
|
||||
"reward_version", "algorithm", "problem",
|
||||
"obs_dim", "act_dim")
|
||||
missing = [key for key in required if key not in metadata]
|
||||
if missing:
|
||||
raise RuntimeError(
|
||||
f"HRL 高层模型元数据缺少字段 {missing}；请用当前 "
|
||||
f"scripts/train_hrl_high.py 重训。")
|
||||
if metadata.get("feature_version") != FEATURE_VERSION:
|
||||
raise RuntimeError(
|
||||
f"HRL 模型特征版本失配: 期望 {FEATURE_VERSION}，"
|
||||
f"实际 {metadata.get('feature_version')}。请用当前版本重训。")
|
||||
if metadata.get("action_layout_version") != ACTION_LAYOUT_VERSION:
|
||||
raise RuntimeError(
|
||||
f"HRL 模型动作布局版本失配: 期望 {ACTION_LAYOUT_VERSION}，"
|
||||
f"实际 {metadata.get('action_layout_version')}。请重训。")
|
||||
if metadata.get("reward_version") != REWARD_VERSION:
|
||||
raise RuntimeError(
|
||||
f"HRL 模型奖励版本失配: 期望 {REWARD_VERSION}，"
|
||||
f"实际 {metadata.get('reward_version')}。请重训。")
|
||||
else:
|
||||
if metadata.get("feature_version") != FEATURE_VERSION:
|
||||
raise RuntimeError(
|
||||
f"低层模型特征版本失配: 期望 {FEATURE_VERSION}，"
|
||||
f"实际 {metadata.get('feature_version')}。请重训。")
|
||||
if metadata.get("algorithm") != "maskable_ppo":
|
||||
raise RuntimeError("HRL 模型必须是 maskable_ppo 训练产物")
|
||||
if problem is not None and metadata.get("problem") not in (None, problem):
|
||||
raise RuntimeError("模型训练问题编号与当前第三/四问不匹配")
|
||||
model = MaskablePPO.load(str(model_path), device=device)
|
||||
try:
|
||||
loaded_obs = model.observation_space.shape[0]
|
||||
loaded_act = model.action_space.n
|
||||
except Exception as exc:
|
||||
raise RuntimeError(f"模型空间与 HRL 接口不兼容: {model_path}") from exc
|
||||
if loaded_obs != obs_dim or loaded_act != act_dim:
|
||||
raise RuntimeError(
|
||||
f"模型接口失配: 期望 obs={obs_dim}/act={act_dim}，"
|
||||
f"实际 obs={loaded_obs}/act={loaded_act}。请重训。")
|
||||
return model
|
||||
|
||||
|
||||
class HighLevelPolicy:
|
||||
"""高层子目标调度策略：观测+掩码 -> 槽位编号（确定性 argmax）。"""
|
||||
|
||||
def __init__(self, model_path: str | Path, *, device: str = "auto",
|
||||
problem: int | None = None):
|
||||
self.device = _resolve_device(device)
|
||||
self.path = Path(model_path)
|
||||
self.model = _load_maskable(self.path, self.device, obs_dim=HIGH_OBS_DIM,
|
||||
act_dim=N_CANDIDATES, problem=problem)
|
||||
meta_path = self.path.with_suffix('.json')
|
||||
self.metadata = json.loads(meta_path.read_text(encoding='utf-8')) \
|
||||
if meta_path.exists() else {}
|
||||
|
||||
def select(self, observation: np.ndarray, action_masks: np.ndarray) -> int:
|
||||
action, _state = self.model.predict(
|
||||
observation, action_masks=action_masks, deterministic=True)
|
||||
return int(action)
|
||||
|
||||
|
||||
class LowLevelPolicy:
|
||||
"""低层到达策略：相对位姿观测 -> 8 方向（确定性 argmax）。"""
|
||||
|
||||
def __init__(self, model_path: str | Path, *, device: str = "auto"):
|
||||
self.device = _resolve_device(device)
|
||||
self.path = Path(model_path)
|
||||
self.model = _load_maskable(self.path, self.device, obs_dim=LOW_OBS_DIM,
|
||||
act_dim=LOW_NUM_ACTIONS, problem=None)
|
||||
|
||||
def select(self, observation: np.ndarray) -> int:
|
||||
action, _state = self.model.predict(observation, deterministic=True)
|
||||
return int(action)
|
||||
|
||||
|
||||
def default_high_model_path(problem: int) -> Path:
|
||||
if problem not in (3, 4):
|
||||
raise ValueError("problem 必须是 3 或 4")
|
||||
return Path("output/hrl") / f"q{problem}_hrl_high.zip"
|
||||
|
||||
|
||||
def default_low_model_path() -> Path:
|
||||
return Path("output/hrl") / "hrl_low.zip"
|
||||
@@ -1,495 +0,0 @@
|
||||
"""高层调度核心：候选槽位构建、观测、动作掩码与任务执行。
|
||||
|
||||
本模块是 :class:`~mcm26b.planner.hrl_loop.HRLLoop`（运行闭环）与
|
||||
:class:`~mcm26b.hrl.hierarchical_env.HierarchicalEnv`（训练环境）共用的
|
||||
混入层，保证训练-部署特征逐位同源（v1"训练部署特征错位"审核教训）。
|
||||
|
||||
宿主须为 :class:`~mcm26b.planner.bayes_loop.BayesLoop` 派生闭环，提供
|
||||
client/belief/particles/ledger/_record_measure 等感知与执行原语；本层只做
|
||||
三件事——
|
||||
|
||||
1. 把整场调度决策离散化为固定槽位（constants.N_CANDIDATES）：
|
||||
clear(20) + locate(20) + station(49) + fallback(20) + 保留(11)；
|
||||
2. 由可见观测（belief + 网格幸存集 + 覆盖账本）构造观测与动作掩码，
|
||||
绝不读取源真值；
|
||||
3. 执行选中候选并报告事实（虚拟耗时增量、新发现、熵减、清除结果），
|
||||
奖励折算由训练环境负责。
|
||||
|
||||
Q4 定向源语义：no_signal 不能删除空间候选（可能是朝向遮挡），故问题4 的
|
||||
幸存集只按 direction/near 更新；遮挡证据走 blocked_probes 梯子与后备清除
|
||||
槽位，与 DirectionalExtension 的保守口径一致。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass, field
|
||||
from typing import NamedTuple
|
||||
|
||||
import numpy as np
|
||||
|
||||
from ..belief import ChannelStatus
|
||||
from ..config import Vec2
|
||||
from ..protocol.rules import CLEAR_SUCCESS, DIRECTION, NEAR, NO_SIGNAL
|
||||
from ..protocol.timing import move_time
|
||||
from .constants import (CHANNEL_COUNT, CLEAR_OFFSET, COST_SCALE_S,
|
||||
FALLBACK_OFFSET, HIGH_GLOBAL_DIM, HIGH_OBS_DIM,
|
||||
LOCATE_OFFSET, MEASURE_COUNT_SCALE, N_CANDIDATES,
|
||||
PER_CANDIDATE_DIM, PER_CHANNEL_DIM, POS_SCALE_M,
|
||||
RADIUS_SCALE_M, STATION_OFFSET)
|
||||
from ..planner import fallback_clear_cells, safe_disk_radius
|
||||
|
||||
__all__ = ["Candidate", "StepOutcome", "HRLControllerMixin", "point_key"]
|
||||
|
||||
|
||||
def point_key(point: Vec2) -> tuple[int, int]:
|
||||
"""候选点去重键：0.5m 网格取整（与 bayes_loop 同口径）。"""
|
||||
return (int(round(point[0] * 2)), int(round(point[1] * 2)))
|
||||
|
||||
|
||||
class Candidate(NamedTuple):
|
||||
"""一个高层候选子目标。slot 为固定槽位编号，供动作掩码与特征定位。"""
|
||||
|
||||
slot: int
|
||||
kind: str # "clear" | "locate" | "probe" | "station" | "fallback"
|
||||
position: Vec2
|
||||
channel: int | None = None
|
||||
station_index: int | None = None
|
||||
service_s: float = 0.0 # 预计服务耗时（含换频，不含移动）
|
||||
targets: int = 1 # station: 待测频道数；fallback: 格数；其余 1
|
||||
|
||||
|
||||
@dataclass
|
||||
class StepOutcome:
|
||||
"""一次高层决策-执行的客观结果（奖励折算由训练环境负责）。"""
|
||||
|
||||
action: int
|
||||
kind: str | None = None
|
||||
channel: int | None = None
|
||||
invalid: bool = False # 掩码外动作（无掩码消融路径）：未与模拟器交互
|
||||
delta_virtual_s: float = 0.0
|
||||
new_detections: tuple[int, ...] = ()
|
||||
info_gains: dict[int, float] = field(default_factory=dict) # 频道 -> 熵减(log)
|
||||
clear_success: tuple[int, ...] = ()
|
||||
redundant: bool = False
|
||||
position_reached: Vec2 | None = None
|
||||
|
||||
|
||||
class HRLControllerMixin:
|
||||
"""高层调度混入：依赖宿主（HRLLoop）提供 BayesLoop 感知与执行原语。
|
||||
|
||||
宿主须在 ``__init__`` 初始化：``_obs_mec_cache={}``、
|
||||
``_station_pending_map={}``、``_initial_alive={channel: len(pf)}``
|
||||
（观测侧缓存与归一化基准；证书相关计算仍用 pf.mec() 精确值）。
|
||||
"""
|
||||
|
||||
problem: int
|
||||
# 由宿主 BayesLoop.__init__ 初始化；此处声明让 pylint 可见该诊断计数。
|
||||
no_shrink_measures: int = 0
|
||||
|
||||
# ------------------------------------------------------------ 候选构建
|
||||
def build_candidates(self) -> list[Candidate]:
|
||||
"""构建当前全部有效候选（固定槽位；训练/推理/heuristic 专家同源）。
|
||||
|
||||
副作用：缓存每频道证书（``self._cert_cache``）与站点待测集
|
||||
（``self._station_pending_map``）供观测构建复用；调用方须在本
|
||||
函数之后、任何状态变更之前构建观测。
|
||||
"""
|
||||
config = self.config
|
||||
detected = self.belief.channels_by_status(ChannelStatus.DETECTED)
|
||||
by_slot: dict[int, Candidate] = {}
|
||||
cert_cache: dict[int, object] = {}
|
||||
station_map: dict[int, list[int]] = {}
|
||||
|
||||
# --- clear 槽位：证书就绪且未因失败等待补测
|
||||
for channel in detected:
|
||||
cert = self._best_certificate(channel)
|
||||
cert_cache[channel] = cert
|
||||
if cert is not None and cert.certified \
|
||||
and channel not in self._clear_failed:
|
||||
by_slot[CLEAR_OFFSET + channel - 1] = Candidate(
|
||||
CLEAR_OFFSET + channel - 1, "clear", tuple(cert.center),
|
||||
channel=channel,
|
||||
service_s=config.cost.clear_success_s, targets=1)
|
||||
|
||||
# --- locate / probe 槽位：未就绪清除的已发现频道
|
||||
for channel in detected:
|
||||
if CLEAR_OFFSET + channel - 1 in by_slot:
|
||||
continue
|
||||
region = self.belief.candidate_set(channel)
|
||||
if region is None:
|
||||
continue
|
||||
blocked = self.blocked_probes.get(channel, []) \
|
||||
if self.problem == 4 else []
|
||||
if blocked:
|
||||
point = self._hrl_probe_point(region, channel)
|
||||
kind = "probe"
|
||||
else:
|
||||
point = self._select_point(channel)
|
||||
kind = "locate"
|
||||
if point is None:
|
||||
continue
|
||||
by_slot[LOCATE_OFFSET + channel - 1] = Candidate(
|
||||
LOCATE_OFFSET + channel - 1, kind, tuple(point), channel=channel,
|
||||
service_s=config.cost.measure_s + config.cost.switch_s, targets=1)
|
||||
|
||||
# --- station 槽位：尚有 UNKNOWN 待测频道的覆盖站
|
||||
for index, point in enumerate(self.ledger.points):
|
||||
pending = self._station_pending(index)
|
||||
station_map[index] = pending
|
||||
if not pending:
|
||||
continue
|
||||
switches = len(pending) - (1 if self.client.state.channel in pending else 0)
|
||||
by_slot[STATION_OFFSET + index] = Candidate(
|
||||
STATION_OFFSET + index, "station", tuple(point),
|
||||
station_index=index,
|
||||
service_s=(config.cost.measure_s * len(pending)
|
||||
+ config.cost.switch_s * max(switches, 0)),
|
||||
targets=len(pending))
|
||||
|
||||
# --- fallback 槽位（仅问题4）：遮挡梯子耗尽且格数在预算内
|
||||
if self.problem == 4:
|
||||
for channel in detected:
|
||||
blocked = self.blocked_probes.get(channel, [])
|
||||
if len(blocked) < config.search.max_blocked_probes:
|
||||
continue
|
||||
region = self.belief.candidate_set(channel)
|
||||
if region is None:
|
||||
continue
|
||||
cells = fallback_clear_cells(region.polygon, config)
|
||||
if not cells or len(cells) > config.search.fallback_max_cells:
|
||||
continue
|
||||
by_slot[FALLBACK_OFFSET + channel - 1] = Candidate(
|
||||
FALLBACK_OFFSET + channel - 1, "fallback",
|
||||
tuple(region.mec()[0]), channel=channel,
|
||||
service_s=config.cost.clear_fail_s * len(cells),
|
||||
targets=len(cells))
|
||||
|
||||
self._cert_cache = cert_cache
|
||||
self._station_pending_map = station_map
|
||||
self._last_candidates = list(by_slot.values())
|
||||
return self._last_candidates
|
||||
|
||||
def candidate_by_slot(self, candidates: list[Candidate],
|
||||
action: int) -> Candidate | None:
|
||||
for cand in candidates:
|
||||
if cand.slot == int(action):
|
||||
return cand
|
||||
return None
|
||||
|
||||
# ------------------------------------------------------------ 动作掩码
|
||||
def hrl_action_mask(self, candidates: list[Candidate]) -> np.ndarray:
|
||||
"""槽位有效性掩码：无候选槽位恒屏蔽（训练/推理共用）。"""
|
||||
mask = np.zeros(N_CANDIDATES, dtype=bool)
|
||||
for cand in candidates:
|
||||
mask[cand.slot] = True
|
||||
return mask
|
||||
|
||||
# ------------------------------------------------------------ 观测
|
||||
def _hrl_obs_mec(self, channel: int) -> tuple[Vec2, float]:
|
||||
"""观测用 MEC：幸存子采样上的最小包围圆，按 pf.version 缓存。
|
||||
|
||||
仅作策略观测特征（近似 MEU/σ），证书相关计算一律用 pf.mec()
|
||||
精确值——两者分开缓存，互不影响。
|
||||
"""
|
||||
pf = self.particles[channel]
|
||||
cached = self._obs_mec_cache.get(channel)
|
||||
if cached is not None and cached[0] == pf.version:
|
||||
return cached[1], cached[2]
|
||||
points = pf.subsample(600)
|
||||
if not points:
|
||||
return (0.0, 0.0), 0.0
|
||||
from ..geometry.localization import smallest_enclosing_circle
|
||||
center, radius = smallest_enclosing_circle(points)
|
||||
self._obs_mec_cache[channel] = (pf.version, center, radius)
|
||||
return center, radius
|
||||
|
||||
def _hrl_channel_status_index(self, channel: int) -> int:
|
||||
"""六状态编码：UNKNOWN/DETECTED/TRACKING/LOCATED/CLEARED/ABSENT。"""
|
||||
status = self.belief.channels[channel].status
|
||||
if status is ChannelStatus.UNKNOWN:
|
||||
return 0
|
||||
if status is ChannelStatus.CLEARED:
|
||||
return 4
|
||||
if status is ChannelStatus.ABSENT:
|
||||
return 5
|
||||
cert = getattr(self, "_cert_cache", {}).get(channel)
|
||||
if cert is not None and getattr(cert, "certified", False):
|
||||
return 3
|
||||
_, radius = self._hrl_obs_mec(channel)
|
||||
return 1 if radius > 500.0 else 2
|
||||
|
||||
def hrl_observation(self, candidates: list[Candidate]) -> np.ndarray:
|
||||
"""构造高层观测（shape=(HIGH_OBS_DIM,)，训练/推理逐位一致）。"""
|
||||
config = self.config
|
||||
field_cfg = config.field
|
||||
position = self.client.state.position
|
||||
belief = self.belief
|
||||
|
||||
detected_count = len(belief.channels_by_status(ChannelStatus.DETECTED))
|
||||
accounted = sum(1 for ch in belief.channels.values()
|
||||
if ch.status in (ChannelStatus.CLEARED,
|
||||
ChannelStatus.ABSENT))
|
||||
cleared = len(self.cleared)
|
||||
glob = [
|
||||
position[0] / POS_SCALE_M,
|
||||
position[1] / POS_SCALE_M,
|
||||
min(self.client.state.virtual_time_s, field_cfg.max_virtual_duration_s)
|
||||
/ field_cfg.max_virtual_duration_s,
|
||||
cleared / 16.0,
|
||||
accounted / CHANNEL_COUNT,
|
||||
(self.client.state.channel - field_cfg.channel_min)
|
||||
/ max(field_cfg.channel_max - field_cfg.channel_min, 1),
|
||||
float(self.problem == 4),
|
||||
detected_count / 16.0,
|
||||
]
|
||||
assert len(glob) == HIGH_GLOBAL_DIM
|
||||
|
||||
pending_station = np.zeros(CHANNEL_COUNT + 1, dtype=bool)
|
||||
for channels in getattr(self, "_station_pending_map", {}).values():
|
||||
for channel in channels:
|
||||
pending_station[channel] = True
|
||||
|
||||
channel_features: list[float] = []
|
||||
for channel in range(field_cfg.channel_min, field_cfg.channel_max + 1):
|
||||
ch = belief.channels[channel]
|
||||
status6 = [0.0] * 6
|
||||
status6[self._hrl_channel_status_index(channel)] = 1.0
|
||||
pf = self.particles[channel]
|
||||
mec_visible = ch.status is ChannelStatus.DETECTED
|
||||
if mec_visible:
|
||||
center, radius = self._hrl_obs_mec(channel)
|
||||
else:
|
||||
center, radius = (0.0, 0.0), 0.0
|
||||
initial = max(self._initial_alive.get(channel, 1), 1)
|
||||
channel_features.extend([
|
||||
*status6,
|
||||
center[0] / POS_SCALE_M,
|
||||
center[1] / POS_SCALE_M,
|
||||
radius / RADIUS_SCALE_M,
|
||||
math.log(len(pf) + 1) / math.log(initial + 1)
|
||||
if initial > 1 else 0.0,
|
||||
min(len(ch.measurements), MEASURE_COUNT_SCALE * 2)
|
||||
/ MEASURE_COUNT_SCALE / 2,
|
||||
math.hypot(center[0] - position[0], center[1] - position[1])
|
||||
/ POS_SCALE_M if mec_visible else 0.0,
|
||||
len(self.blocked_probes.get(channel, []))
|
||||
/ max(config.search.max_blocked_probes, 1)
|
||||
if self.problem == 4 else 0.0,
|
||||
1.0 if (ch.status is ChannelStatus.UNKNOWN
|
||||
and pending_station[channel]) else 0.0,
|
||||
])
|
||||
assert len(channel_features) == CHANNEL_COUNT * PER_CHANNEL_DIM
|
||||
|
||||
by_slot = {cand.slot: cand for cand in candidates}
|
||||
candidate_features: list[float] = []
|
||||
for slot in range(N_CANDIDATES):
|
||||
cand = by_slot.get(slot)
|
||||
if cand is None:
|
||||
candidate_features.extend([0.0] * PER_CANDIDATE_DIM)
|
||||
continue
|
||||
dx, dy = cand.position[0] - position[0], cand.position[1] - position[1]
|
||||
move_s = move_time(position, cand.position, config.cost)
|
||||
candidate_features.extend([
|
||||
1.0 if cand.kind == "clear" else 0.0,
|
||||
1.0 if cand.kind in ("locate", "probe") else 0.0,
|
||||
1.0 if cand.kind == "station" else 0.0,
|
||||
1.0 if cand.kind == "fallback" else 0.0,
|
||||
dx / POS_SCALE_M,
|
||||
dy / POS_SCALE_M,
|
||||
min((move_s + cand.service_s) / COST_SCALE_S, 2.0),
|
||||
min(cand.targets / 16.0, 1.0),
|
||||
])
|
||||
assert len(candidate_features) == N_CANDIDATES * PER_CANDIDATE_DIM
|
||||
|
||||
observation = np.asarray(glob + channel_features + candidate_features,
|
||||
dtype=np.float32)
|
||||
assert observation.shape == (HIGH_OBS_DIM,)
|
||||
return observation
|
||||
|
||||
# ------------------------------------------------------------ 启发式专家
|
||||
def heuristic_action(self, candidates: list[Candidate]) -> int:
|
||||
"""把 BayesLoop 任务排序映射为槽位（BC 专家 + 无策略降级路径）。
|
||||
|
||||
任务表与排序完全复用 :meth:`BayesLoop._build_tasks` /
|
||||
:meth:`BayesLoop._order_tasks`；映射失败（如 Q4 梯子任务不在
|
||||
任务表中）退化为最近有效候选。
|
||||
"""
|
||||
by_slot = {cand.slot: cand for cand in candidates}
|
||||
if not by_slot:
|
||||
raise RuntimeError("heuristic_action：当前无可用候选")
|
||||
try:
|
||||
order = self._order_tasks(self._build_tasks())
|
||||
first = order[0]
|
||||
if first.kind == "clear" and first.channel is not None:
|
||||
slot = CLEAR_OFFSET + first.channel - 1
|
||||
elif first.kind == "locate" and first.channel is not None:
|
||||
slot = LOCATE_OFFSET + first.channel - 1
|
||||
else:
|
||||
slot = STATION_OFFSET + (first.station_index or 0)
|
||||
if slot in by_slot:
|
||||
return slot
|
||||
except Exception:
|
||||
pass
|
||||
position = self.client.state.position
|
||||
return min(by_slot, key=lambda s: (
|
||||
math.dist(position, by_slot[s].position), s))
|
||||
|
||||
# ------------------------------------------------------------ 决策执行
|
||||
def hrl_step(self, action: int,
|
||||
candidates: list[Candidate] | None = None) -> StepOutcome:
|
||||
"""执行一次高层决策；candidates 为空则现场构建（训练环境逐位同源）。"""
|
||||
action = int(action)
|
||||
self._reset_step_recording()
|
||||
old_virtual = self.client.state.virtual_time_s
|
||||
if candidates is None:
|
||||
candidates = self.build_candidates()
|
||||
cand = self.candidate_by_slot(candidates, action)
|
||||
if cand is None:
|
||||
return StepOutcome(action=action, invalid=True)
|
||||
if self.stop_reason is not None:
|
||||
return StepOutcome(action=action, kind=cand.kind,
|
||||
channel=cand.channel, invalid=True)
|
||||
|
||||
if cand.kind == "station":
|
||||
self._execute_station(cand.station_index)
|
||||
elif cand.kind in ("locate", "probe"):
|
||||
self._hrl_execute_locate(cand)
|
||||
elif cand.kind == "clear":
|
||||
self._hrl_execute_clear(cand)
|
||||
elif cand.kind == "fallback":
|
||||
self._hrl_execute_fallback(cand)
|
||||
else: # pragma: no cover - 布局守卫
|
||||
raise ValueError(f"未知候选类型: {cand.kind}")
|
||||
|
||||
outcome = StepOutcome(action=action, kind=cand.kind,
|
||||
channel=cand.channel,
|
||||
position_reached=tuple(self.client.state.position))
|
||||
outcome.delta_virtual_s = self.client.state.virtual_time_s - old_virtual
|
||||
new_detections: list[int] = []
|
||||
info_gains: dict[int, float] = {}
|
||||
redundant = False
|
||||
for event in self._step_events:
|
||||
if event["was_unknown"] and event["result"] in (DIRECTION, NEAR):
|
||||
new_detections.append(event["channel"])
|
||||
gain = math.log10(event["pf_before"] + 1) - math.log10(event["pf_after"] + 1)
|
||||
info_gains[event["channel"]] = (info_gains.get(event["channel"], 0.0)
|
||||
+ min(max(gain, 0.0), 2.0))
|
||||
redundant = redundant or event["redundant"]
|
||||
outcome.new_detections = tuple(new_detections)
|
||||
outcome.info_gains = info_gains
|
||||
outcome.redundant = redundant
|
||||
outcome.clear_success = tuple(self._step_clears)
|
||||
self._step_clears = []
|
||||
return outcome
|
||||
|
||||
# ------------------------------------------------------------ 执行原语
|
||||
def _hrl_execute_locate(self, cand: Candidate) -> None:
|
||||
"""定位测量（含 Q4 梯子探点）：记录证据，不自动清除（高层决策）。"""
|
||||
config = self.config
|
||||
channel = cand.channel
|
||||
pf = self.particles[channel]
|
||||
before = len(pf)
|
||||
outcome = self.client.measure(cand.position, channel)
|
||||
self._record_measure(channel, cand.position, outcome)
|
||||
self._may_clear.add(channel)
|
||||
if self.problem == 4:
|
||||
if outcome.result == NO_SIGNAL:
|
||||
if channel not in self.directional_observed:
|
||||
region = self.belief.candidate_set(channel)
|
||||
safe = safe_disk_radius(region, config) if region else -1.0
|
||||
if safe >= 0 and math.dist(cand.position, region.mec()[0]) <= safe:
|
||||
self.directional_observed.append(channel)
|
||||
self.blocked_probes.setdefault(channel, []).append(cand.position)
|
||||
else:
|
||||
self.blocked_probes.pop(channel, None)
|
||||
if len(pf) == before:
|
||||
self._stall[channel] = self._stall.get(channel, 0) + 1
|
||||
self.no_shrink_measures += 1
|
||||
else:
|
||||
self._stall[channel] = 0
|
||||
|
||||
def _hrl_execute_clear(self, cand: Candidate) -> None:
|
||||
"""证书清除：证书失效（不应被掩码允许）时静默跳过。"""
|
||||
cert = self._best_certificate(cand.channel)
|
||||
if cert is None or not cert.certified:
|
||||
self._anomaly("hrl_clear_without_certificate", cand.channel)
|
||||
return
|
||||
if self._attempt_clear(cand.channel, cert.center, cert):
|
||||
self._clear_failed.discard(cand.channel)
|
||||
self._stall.pop(cand.channel, None)
|
||||
else:
|
||||
self._clear_failed.add(cand.channel)
|
||||
self._may_clear.discard(cand.channel)
|
||||
|
||||
def _hrl_execute_fallback(self, cand: Candidate) -> None:
|
||||
"""Q4 后备网格清除（与 DirectionalExtension._fallback_clear 同口径）。"""
|
||||
channel = cand.channel
|
||||
region = self.belief.candidate_set(channel)
|
||||
if region is None:
|
||||
return
|
||||
config, session = self.config, self.config.session
|
||||
cells = fallback_clear_cells(region.polygon, config)
|
||||
entry = {"channel": channel, "cells": len(cells)}
|
||||
self.fallback_log.append(entry)
|
||||
if len(cells) > config.search.fallback_max_cells:
|
||||
entry["outcome"] = "skipped_cell_cap"
|
||||
return
|
||||
remaining = self.client.real_remaining_s
|
||||
if remaining is not None and len(cells) * session.real_rtt_p95_s \
|
||||
+ session.reserve_exit_s > remaining:
|
||||
entry["outcome"] = "skipped_budget"
|
||||
return
|
||||
remaining_cells = list(cells)
|
||||
attempts = 0
|
||||
while remaining_cells:
|
||||
if self._virtual_cap_reached():
|
||||
entry["outcome"] = "skipped_virtual_budget"
|
||||
entry["attempts"] = attempts
|
||||
return
|
||||
cell = min(remaining_cells, key=lambda c: (
|
||||
math.dist(self.client.state.position, c), c))
|
||||
remaining_cells.remove(cell)
|
||||
attempts += 1
|
||||
outcome = self.client.clear(cell, channel)
|
||||
if outcome.result == CLEAR_SUCCESS:
|
||||
self.belief.mark_cleared(channel)
|
||||
self.cleared.append(channel)
|
||||
self._step_clears.append(channel)
|
||||
self.blocked_probes.pop(channel, None)
|
||||
entry["outcome"] = "cleared"
|
||||
entry["attempts"] = attempts
|
||||
return
|
||||
entry["outcome"] = "exhausted"
|
||||
entry["attempts"] = attempts
|
||||
self._anomaly("hrl_fallback_exhausted", channel, cells=len(cells))
|
||||
|
||||
def _hrl_probe_point(self, region, channel: int) -> Vec2:
|
||||
"""定向遮挡梯子探点（与 DirectionalExtension._next_probe 同口径）。"""
|
||||
from ..planner import select_measurement_point
|
||||
blocked = self.blocked_probes.get(channel, [])
|
||||
center, radius = region.mec()
|
||||
if not blocked:
|
||||
return select_measurement_point(region, self.client.state.position,
|
||||
self.config).position
|
||||
safe = safe_disk_radius(region, self.config)
|
||||
first = blocked[0]
|
||||
if len(blocked) == 1 and safe >= 0:
|
||||
dx, dy = center[0] - first[0], center[1] - first[1]
|
||||
norm = math.hypot(dx, dy)
|
||||
if norm <= 1e-9:
|
||||
return (center[0] + safe, center[1])
|
||||
scale = min(1.0, safe / norm)
|
||||
return (center[0] + dx * scale, center[1] + dy * scale)
|
||||
base = math.atan2(first[1] - center[1], first[0] - center[0])
|
||||
azimuth = base + math.radians(45.0 * (len(blocked) - 1))
|
||||
return (center[0] + radius * math.cos(azimuth),
|
||||
center[1] + radius * math.sin(azimuth))
|
||||
|
||||
# ------------------------------------------------------------ 事件捕获
|
||||
def _reset_step_recording(self) -> None:
|
||||
self._step_events = []
|
||||
self._step_clears = []
|
||||
|
||||
def _record_event(self, **event) -> None:
|
||||
self._step_events.append(event)
|
||||
@@ -1,172 +0,0 @@
|
||||
"""P7 分层强化学习闭环（``--strategy hrl`` / ``--q4-strategy hrl``）。
|
||||
|
||||
边界与保守性声明（与 rl_loop.py 同口径）：
|
||||
- 高层只在固定候选槽位上做调度决策（证书清除 / 信息增益定位 / 覆盖站 /
|
||||
Q4 梯子探点与后备清除），候选几何全部由既有规则生成；
|
||||
- 感知（belief + 网格幸存集）、清除证书、跳测规则、异常留痕完全继承
|
||||
:class:`BayesLoop`，不触碰真源保留定理的前提；
|
||||
- 问题3 的幸存集按全部读数更新（no_signal 删盘）；问题4 定向源的
|
||||
no_signal 不能排除朝向遮挡，幸存集只按 direction/near 更新，遮挡
|
||||
证据走 blocked_probes 梯子与后备清除槽位（DirectionalExtension 口径）；
|
||||
- 高层策略缺失时经 :meth:`heuristic_action` 降级为 BayesLoop 任务排序，
|
||||
闭环仍然完备可运行。
|
||||
|
||||
问题4 发现覆盖使用 49 点网格（七点环的覆盖证明只对全向源成立）。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from ..belief import ChannelStatus
|
||||
from ..config import ProblemConfig, Vec2
|
||||
from ..coverage import CoverageLedger, coverage_points
|
||||
from ..hrl.scheduler import HRLControllerMixin, point_key
|
||||
from ..protocol.rules import NO_SIGNAL
|
||||
from .bayes_loop import BayesLoop
|
||||
|
||||
|
||||
class HRLLoop(HRLControllerMixin, BayesLoop):
|
||||
"""分层闭环：BayesLoop 感知/证书 + 高层槽位调度（问题3/4 共用）。"""
|
||||
|
||||
def __init__(self, client, config: ProblemConfig = ProblemConfig(), *,
|
||||
problem: int = 3, high_policy=None, low_policy=None,
|
||||
trace_executor: bool = False):
|
||||
super().__init__(client, config)
|
||||
if problem not in (3, 4):
|
||||
raise ValueError("problem 必须是 3 或 4")
|
||||
self.problem = int(problem)
|
||||
self.high_policy = high_policy
|
||||
self.low_policy = low_policy
|
||||
self.trace_executor = bool(trace_executor)
|
||||
self.executor_trace: list[dict] = []
|
||||
self.hrl_decisions: list[dict] = []
|
||||
self._step_events: list[dict] = []
|
||||
self._step_clears: list[int] = []
|
||||
self._cert_cache: dict = {}
|
||||
self._last_candidates: list = []
|
||||
self._obs_mec_cache: dict = {}
|
||||
self._station_pending_map: dict = {}
|
||||
# 观测里幸存格数的归一化基准（初始全集），使特征对训练/部署的
|
||||
# 网格密度差不变。
|
||||
self._initial_alive = {channel: len(pf)
|
||||
for channel, pf in self.particles.items()}
|
||||
if problem == 4:
|
||||
self.directional_observed: list[int] = []
|
||||
self.blocked_probes: dict[int, list[Vec2]] = {}
|
||||
self.fallback_log: list[dict] = []
|
||||
# 49 点网格：七点环覆盖证明只对全向源成立 [口径见模块注释]
|
||||
self.ledger = CoverageLedger(config, points=coverage_points(config))
|
||||
|
||||
# ------------------------------------------------------------ 主循环
|
||||
def _coverage_phase(self) -> None:
|
||||
"""统一任务循环：每步构建候选，由高层策略（或启发式降级）选槽执行。"""
|
||||
while self.stop_reason is None:
|
||||
if self._count_limit_reached() or self._virtual_cap_reached():
|
||||
return
|
||||
# 顺路观测会改变信念、证书和候选；必须发生在本轮快照之前。
|
||||
if self.high_policy is None and self.problem == 3 \
|
||||
and self.config.bayes.opportunistic_measure:
|
||||
self._measure_at_stop()
|
||||
if self.stop_reason is not None or self._count_limit_reached() \
|
||||
or self._virtual_cap_reached():
|
||||
return
|
||||
candidates = self.build_candidates()
|
||||
if not candidates:
|
||||
return
|
||||
if self.high_policy is not None:
|
||||
observation = self.hrl_observation(candidates)
|
||||
mask = self.hrl_action_mask(candidates)
|
||||
action = self.high_policy.select(observation, mask)
|
||||
else:
|
||||
# 降级路径：BayesLoop 任务排序（无 RL 依赖时闭环仍完备）
|
||||
action = self.heuristic_action(candidates)
|
||||
if self.candidate_by_slot(candidates, action) is None:
|
||||
# 非法策略输出不能造成零耗时无限循环；事实记入异常再安全调度。
|
||||
self._anomaly("hrl_invalid_policy_action", 0, action=int(action))
|
||||
action = self.heuristic_action(candidates)
|
||||
cand = self.candidate_by_slot(candidates, action)
|
||||
if cand is not None:
|
||||
self._log_low_level(cand)
|
||||
self.hrl_decisions.append({
|
||||
"action": int(action), "kind": cand.kind,
|
||||
"channel": cand.channel,
|
||||
"virtual_time_s": self.client.state.virtual_time_s})
|
||||
self.hrl_step(action, candidates=candidates)
|
||||
|
||||
# ------------------------------------------------------------ 感知记录
|
||||
def _record_measure(self, channel: int, position: Vec2, outcome) -> None:
|
||||
"""BayesLoop._record_measure 的事件捕获版；问题4 no_signal 不删格。
|
||||
|
||||
Q4 语义：no_signal 可能是朝向遮挡 [观测推理]，删除幸存格会破坏
|
||||
真源保留，因此问题4 幸存集只按 direction/near 更新；缺席证明
|
||||
交由 49 点网格覆盖账本（继承的 _mark_absent_after_coverage）。
|
||||
"""
|
||||
key = point_key(position)
|
||||
redundant = key in self._measured_points.get(channel, set())
|
||||
belief = self.belief.channels[channel]
|
||||
was_unknown = belief.status is ChannelStatus.UNKNOWN
|
||||
pf = self.particles[channel]
|
||||
before = len(pf)
|
||||
self.belief.update_measure(channel, position, outcome.result,
|
||||
outcome.svd_deg)
|
||||
self._measured_points.setdefault(channel, set()).add(key)
|
||||
self._clear_failed.discard(channel)
|
||||
belief = self.belief.channels[channel]
|
||||
if belief.status not in (ChannelStatus.CLEARED, ChannelStatus.ABSENT):
|
||||
if not (self.problem == 4 and outcome.result == NO_SIGNAL):
|
||||
pf.apply_measure(position, outcome.result, outcome.svd_deg)
|
||||
if not pf:
|
||||
if belief.status is ChannelStatus.UNKNOWN:
|
||||
self.belief.mark_absent(channel)
|
||||
if channel not in self.pf_absent_channels:
|
||||
self.pf_absent_channels.append(channel)
|
||||
else:
|
||||
self._anomaly("pf_empty_while_detected", channel,
|
||||
position=position)
|
||||
self._record_event(channel=channel, result=outcome.result,
|
||||
pf_before=before, pf_after=len(pf),
|
||||
redundant=redundant, was_unknown=was_unknown)
|
||||
|
||||
def _attempt_clear(self, channel: int, target: Vec2, cert) -> bool:
|
||||
success = super()._attempt_clear(channel, target, cert)
|
||||
if success and hasattr(self, "blocked_probes"):
|
||||
self.blocked_probes.pop(channel, None)
|
||||
if success:
|
||||
self._step_clears.append(channel)
|
||||
return success
|
||||
|
||||
# ------------------------------------------------------------ 低层执行留痕
|
||||
def _log_low_level(self, cand) -> None:
|
||||
"""低层到达推演留痕（本地步进，不与模拟器交互，不产生虚拟耗时）。"""
|
||||
if not self.trace_executor:
|
||||
return
|
||||
from ..hrl.executor import LowLevelExecutor
|
||||
executor = getattr(self, "_executor", None)
|
||||
if executor is None:
|
||||
executor = self._executor = LowLevelExecutor(self.low_policy)
|
||||
waypoints = executor.roll(self.client.state.position, cand.position)
|
||||
if self.low_policy is not None:
|
||||
self.executor_trace.append({
|
||||
"kind": cand.kind, "channel": cand.channel,
|
||||
"goal": [round(v, 2) for v in cand.position],
|
||||
"waypoints": [[round(v, 2) for v in p] for p in waypoints],
|
||||
"navigation": dict(executor.last_roll_info),
|
||||
})
|
||||
|
||||
# ------------------------------------------------------------ 收尾
|
||||
def _summary(self, info) -> dict:
|
||||
report = super()._summary(info)
|
||||
report["strategy"] = "hrl_q4" if self.problem == 4 else "hrl"
|
||||
report["hrl_problem"] = self.problem
|
||||
kinds: dict[str, int] = {}
|
||||
for decision in self.hrl_decisions:
|
||||
kinds[decision["kind"]] = kinds.get(decision["kind"], 0) + 1
|
||||
report["hrl_action_kinds"] = kinds
|
||||
report["hrl_decisions"] = self.hrl_decisions
|
||||
report["hrl_executor_trace"] = self.executor_trace
|
||||
if self.problem == 4:
|
||||
report["directional_observed_channels"] = \
|
||||
list(self.directional_observed)
|
||||
report["fallback_clears"] = self.fallback_log
|
||||
report["fallback_clear_attempts"] = sum(
|
||||
entry.get("attempts", 0) for entry in self.fallback_log)
|
||||
return report
|
||||
@@ -1,94 +0,0 @@
|
||||
"""P6b 强化学习闭环接入：PPO 学习"下一检测点"选择（问题3/4）。
|
||||
|
||||
边界与保守性声明：
|
||||
- RL 只替换测量点排序（在既有几何候选集内做 argmax），不生成新的
|
||||
空间候选、不修改覆盖证明、不修改清除证书；
|
||||
- 问题3：继承 :class:`BayesLoop`（七点覆盖 + 粒子滤波 + 滚动任务调度），
|
||||
仅 :meth:`_select_point` 改由 PPO 输出候选编号；
|
||||
- 问题4：继承 :class:`DirectionalExtension`（49 点定向覆盖 + 对跖/环探 +
|
||||
20m 有限圆盘覆盖后备清除），仅 :meth:`_next_probe` 改由 PPO 输出；
|
||||
遮挡计数、定向观测与后备清除兜底完全保留；
|
||||
- 训练环境与运行特征同源（mcm26b.rl.features），模型加载时校验
|
||||
观测/动作维度，不匹配即拒绝运行，避免静默错位。
|
||||
|
||||
默认模型位置：output/rl/q3_ppo.zip、output/rl/q4_ppo.zip。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from ..config import ProblemConfig, Vec2
|
||||
from ..rl.policy import PointSelectionPolicy, default_model_path
|
||||
from . import DirectionalExtension
|
||||
from .bayes_loop import BayesLoop
|
||||
|
||||
|
||||
class RLBayesLoop(BayesLoop):
|
||||
"""问题3：贝叶斯闭环骨架 + PPO 定位选点（--strategy rl）。"""
|
||||
|
||||
def __init__(self, client, config: ProblemConfig = ProblemConfig(), *,
|
||||
model_path: str | Path | None = None, device: str = "auto"):
|
||||
super().__init__(client, config)
|
||||
self.rl_model_path = Path(model_path) if model_path else default_model_path(3)
|
||||
self.rl_policy = PointSelectionPolicy(self.rl_model_path, device=device)
|
||||
self.rl_probe_actions: list[dict] = []
|
||||
|
||||
def _select_point(self, channel: int) -> Vec2 | None:
|
||||
region = self.belief.candidate_set(channel)
|
||||
if region is None:
|
||||
return None
|
||||
point, action = self.rl_policy.select(
|
||||
region, self.client.state.position, channel, self.config,
|
||||
problem=3, blocked=[], belief=self.belief.channels[channel],
|
||||
virtual_time_s=self.client.state.virtual_time_s,
|
||||
receiver_channel=self.client.state.channel)
|
||||
self.rl_probe_actions.append({
|
||||
"channel": channel, "action": int(action),
|
||||
"position": [round(point[0], 3), round(point[1], 3)],
|
||||
"virtual_time_s": self.client.state.virtual_time_s,
|
||||
})
|
||||
return point
|
||||
|
||||
def _summary(self, info) -> dict:
|
||||
report = super()._summary(info)
|
||||
report["strategy"] = "rl"
|
||||
report["rl_model"] = str(self.rl_model_path)
|
||||
report["rl_device"] = self.rl_policy.device
|
||||
report["rl_probe_actions"] = self.rl_probe_actions
|
||||
return report
|
||||
|
||||
|
||||
class RLDirectionalLoop(DirectionalExtension):
|
||||
"""问题4：定向闭环骨架 + PPO 遮挡梯子选点（--q4-strategy rl）。"""
|
||||
|
||||
def __init__(self, client, config: ProblemConfig = ProblemConfig(), *,
|
||||
model_path: str | Path | None = None, device: str = "auto"):
|
||||
super().__init__(client, config)
|
||||
self.rl_model_path = Path(model_path) if model_path else default_model_path(4)
|
||||
self.rl_policy = PointSelectionPolicy(self.rl_model_path, device=device)
|
||||
self.rl_probe_actions: list[dict] = []
|
||||
|
||||
def _next_probe(self, region, channel: int) -> Vec2:
|
||||
blocked = self.blocked_probes.get(channel, [])
|
||||
point, action = self.rl_policy.select(
|
||||
region, self.client.state.position, channel, self.config,
|
||||
problem=4, blocked=blocked,
|
||||
belief=self.belief.channels[channel],
|
||||
virtual_time_s=self.client.state.virtual_time_s,
|
||||
receiver_channel=self.client.state.channel)
|
||||
self.rl_probe_actions.append({
|
||||
"channel": channel, "action": int(action),
|
||||
"position": [round(point[0], 3), round(point[1], 3)],
|
||||
"blocked_count": len(blocked),
|
||||
"virtual_time_s": self.client.state.virtual_time_s,
|
||||
})
|
||||
return point
|
||||
|
||||
def _summary(self, info) -> dict:
|
||||
report = super()._summary(info)
|
||||
report["strategy"] = "rl_q4"
|
||||
report["rl_model"] = str(self.rl_model_path)
|
||||
report["rl_device"] = self.rl_policy.device
|
||||
report["rl_probe_actions"] = self.rl_probe_actions
|
||||
return report
|
||||
@@ -1,175 +0,0 @@
|
||||
"""跨平台子进程控制：启动隔离、协作式暂停/停止、必要时强杀整棵进程树。
|
||||
|
||||
训练前端要在 POSIX 与 Windows 上表现一致，而两边的进程模型差别正好卡在
|
||||
「停止训练」这条路径上：
|
||||
|
||||
- ``os.killpg`` / ``os.getpgid`` **只存在于 POSIX**；Windows 上访问它们会抛
|
||||
``AttributeError``。旧实现只捕获 ``ProcessLookupError`` / ``PermissionError``，
|
||||
于是 ``POST /api/train/stop`` 在 Windows 上直接 500，训练进程照跑——表现就是
|
||||
「点停止没反应」。
|
||||
- ``subprocess.Popen(start_new_session=True)`` 在 Windows 上被**静默忽略**：
|
||||
CPython 的 Windows ``_execute_child`` 形参名就是 ``unused_start_new_session``，
|
||||
要隔离进程组只能传 ``CREATE_NEW_PROCESS_GROUP``。
|
||||
- POSIX 的 SIGTERM 在 Windows 上经 ``os.kill`` 会退化成 ``TerminateProcess``：
|
||||
硬杀，训练脚本里"收到信号后保存模型"的处理器根本不会执行。
|
||||
|
||||
所以「优雅暂停 / 优雅停止」不依赖信号量语义，而是走一个**控制文件**：前端原子
|
||||
写 JSON，训练脚本每步轮询。两端都只是普通文件 IO，Windows 与 POSIX 行为一致；
|
||||
信号与强杀退居兜底，只在训练进程不响应控制文件时使用。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
__all__ = [
|
||||
"IS_WINDOWS", "ControlState", "spawn_kwargs", "write_control",
|
||||
"read_control", "write_json_atomic", "terminate_process_tree",
|
||||
]
|
||||
|
||||
# 运行期常量而非"导入期分支"：调用方（含测试）可以 monkeypatch 它来模拟 Windows
|
||||
IS_WINDOWS = os.name == "nt"
|
||||
|
||||
# 只有 Windows 定义 CREATE_NEW_PROCESS_GROUP；POSIX 上取 0 表示不使用 creationflags
|
||||
_CREATE_NEW_PROCESS_GROUP = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)
|
||||
|
||||
# 发出「停止」到强杀之间留给训练脚本收尾（保存模型 + 写元数据）的秒数
|
||||
DEFAULT_GRACE_S = 15.0
|
||||
|
||||
|
||||
def spawn_kwargs() -> dict[str, Any]:
|
||||
"""返回 ``Popen`` 的平台相关参数：让训练进程独立于前端进程组。
|
||||
|
||||
POSIX 用 ``setsid``（``start_new_session``）；Windows 上用
|
||||
``CREATE_NEW_PROCESS_GROUP``，否则前端所在控制台的 Ctrl+C 会连带打到训练进程。
|
||||
"""
|
||||
if IS_WINDOWS:
|
||||
if _CREATE_NEW_PROCESS_GROUP:
|
||||
return {"creationflags": _CREATE_NEW_PROCESS_GROUP}
|
||||
return {}
|
||||
return {"start_new_session": True}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ControlState:
|
||||
"""控制文件的一次快照；缺失或损坏时字段全为默认（即"无指令"）。"""
|
||||
|
||||
paused: bool = False
|
||||
stop: bool = False
|
||||
seq: int = 0
|
||||
ts: float = 0.0
|
||||
|
||||
|
||||
def write_control(path: Path, *, paused: bool = False, stop: bool = False,
|
||||
seq: int | None = None) -> ControlState:
|
||||
"""原子写控制文件（同目录临时文件 + ``os.replace``）。
|
||||
|
||||
训练侧随时可能来读，必须避免读到写了一半的 JSON。``os.replace`` 在
|
||||
Windows 与 POSIX 上都是原子的（同目录、同卷）。
|
||||
"""
|
||||
path = Path(path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
state = ControlState(
|
||||
paused=bool(paused),
|
||||
stop=bool(stop),
|
||||
seq=int(seq) if seq is not None else time.time_ns(),
|
||||
ts=time.time(),
|
||||
)
|
||||
payload = {"paused": state.paused, "stop": state.stop,
|
||||
"seq": state.seq, "ts": state.ts}
|
||||
tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp")
|
||||
tmp.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
|
||||
os.replace(tmp, path)
|
||||
return state
|
||||
|
||||
|
||||
def read_control(path: Path) -> ControlState:
|
||||
"""读控制文件；缺失 / 半截 / 损坏一律当作"无指令"，绝不让训练崩在控制面上。"""
|
||||
try:
|
||||
raw = Path(path).read_text(encoding="utf-8")
|
||||
payload = json.loads(raw)
|
||||
except (OSError, ValueError, UnicodeDecodeError):
|
||||
return ControlState()
|
||||
if not isinstance(payload, dict):
|
||||
return ControlState()
|
||||
return ControlState(
|
||||
paused=bool(payload.get("paused", False)),
|
||||
stop=bool(payload.get("stop", False)),
|
||||
seq=int(payload.get("seq") or 0),
|
||||
ts=float(payload.get("ts") or 0.0),
|
||||
)
|
||||
|
||||
|
||||
def write_json_atomic(path: Path, payload: Any) -> None:
|
||||
"""原子写 JSON 文件（同目录临时文件 + ``os.replace``）。
|
||||
|
||||
用于"急停恢复"相关的标记文件（``resume.json`` / ``forced_stop.json`` 等）：
|
||||
读取方随时可能来读，必须避免看到写了一半的内容。
|
||||
"""
|
||||
path = Path(path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp")
|
||||
tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8")
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
def terminate_process_tree(process: subprocess.Popen | None, *,
|
||||
grace_s: float = DEFAULT_GRACE_S,
|
||||
poll_s: float = 0.2) -> bool:
|
||||
"""等待 ``grace_s`` 让协作式停止生效，超时则强杀整棵进程树。
|
||||
|
||||
返回**是否动用了强杀**：``False`` 表示进程在宽限期内自行退出（模型有机会
|
||||
落盘），``True`` 表示进程被强杀、最后一段产物可能缺失。调用方应先通过控制
|
||||
文件请求停止，再调用本函数；本函数不负责写控制文件。
|
||||
"""
|
||||
if process is None:
|
||||
return False
|
||||
if process.poll() is not None:
|
||||
return False
|
||||
deadline = time.monotonic() + max(0.0, grace_s)
|
||||
while time.monotonic() < deadline:
|
||||
if process.poll() is not None:
|
||||
return False
|
||||
time.sleep(min(poll_s, max(0.0, deadline - time.monotonic())))
|
||||
if process.poll() is not None:
|
||||
return False
|
||||
_hard_kill_tree(process)
|
||||
try:
|
||||
process.wait(timeout=10.0)
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
return True
|
||||
|
||||
|
||||
def _hard_kill_tree(process: subprocess.Popen) -> None:
|
||||
"""强杀进程树：POSIX 走进程组 SIGKILL，Windows 走 ``taskkill /T``。
|
||||
|
||||
这里所有 POSIX 专属调用都走 ``getattr`` 探测，Windows 上拿不到分支就退回
|
||||
``Popen.kill()``——不会再出现 ``module 'os' has no attribute 'killpg'``。
|
||||
"""
|
||||
if IS_WINDOWS:
|
||||
try:
|
||||
subprocess.run(["taskkill", "/F", "/T", "/PID", str(process.pid)],
|
||||
capture_output=True, check=False)
|
||||
return
|
||||
except OSError:
|
||||
process.kill()
|
||||
return
|
||||
killpg = getattr(os, "killpg", None)
|
||||
getpgid = getattr(os, "getpgid", None)
|
||||
sigkill = getattr(signal, "SIGKILL", None)
|
||||
if killpg is not None and getpgid is not None and sigkill is not None:
|
||||
try:
|
||||
killpg(getpgid(process.pid), sigkill)
|
||||
return
|
||||
except (ProcessLookupError, PermissionError, OSError):
|
||||
pass
|
||||
process.kill()
|
||||
@@ -1,14 +0,0 @@
|
||||
"""问题3/4 强化学习扩展（可选依赖组 ``rl``）。
|
||||
|
||||
- features.py：观测/候选动作的纯几何接口（numpy），训练与运行共享；
|
||||
- env.py：单源定位 Gymnasium 训练环境（依赖 referee，仅训练侧加载）；
|
||||
- policy.py：PPO 模型加载与确定性推理（惰性加载 torch/sb3）。
|
||||
|
||||
运行闭环默认不导入 torch/stable-baselines3；只有 CLI 显式选择
|
||||
``--strategy rl``（问题3）或 ``--q4-strategy rl``（问题4）时，
|
||||
``planner.rl_loop`` 才构造 :class:`policy.PointSelectionPolicy`。
|
||||
"""
|
||||
|
||||
from .features import candidate_actions, build_observation
|
||||
|
||||
__all__ = ["candidate_actions", "build_observation"]
|
||||
@@ -1,11 +0,0 @@
|
||||
"""RL 观测/动作空间常量：模型接口的唯一字面量来源。
|
||||
|
||||
布局说明见 features.py；这些数字由训练脚本写入模型接口，
|
||||
改动其中任一值都会使已训练模型失配，须同步重训。
|
||||
"""
|
||||
|
||||
GLOBAL_DIM = 41
|
||||
PER_ACTION_DIM = 6
|
||||
NUM_ACTIONS = 60
|
||||
LADDER_SLOTS = 8
|
||||
OBS_DIM = GLOBAL_DIM + NUM_ACTIONS * PER_ACTION_DIM # 401
|
||||
@@ -1,331 +0,0 @@
|
||||
"""问题3/4 的强化学习训练环境：单源"测向—定位—清除"子任务。
|
||||
|
||||
环境与自建评测机 referee 共享同一边界（内存回环，与真实 HTTP 同协议），
|
||||
观测只来自 :class:`mcm26b.belief.BeliefState` 与协议响应；真值仅在 reset
|
||||
时用于布设课程场景，不进入观测。策略端看到的接口与闭环运行时一致。
|
||||
|
||||
问题抽象：机器狗已获得某频道首次 direction 读数，需要在一组几何可行候选
|
||||
检测点中选择下一个测点，目标是尽快把候选集最小包围圆压到 20m 清除证书
|
||||
阈值并清除成功。问题4额外引入定向源遮挡：保证接收盘内出现 no_signal 是
|
||||
"朝向遮挡"的观测证据，环境记录 blocked 序列并开放环探候选。
|
||||
|
||||
该环境是训练侧组件（依赖 referee + gymnasium），运行时代码不导入本模块；
|
||||
导入依赖由可选组 ``[dependency-groups] rl`` 提供。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from random import Random
|
||||
|
||||
import gymnasium as gym
|
||||
import numpy as np
|
||||
from gymnasium import spaces
|
||||
|
||||
from ..belief import BeliefState
|
||||
from ..config import ProblemConfig
|
||||
from ..planner import clear_certificate, fallback_clear_cells, safe_disk_radius
|
||||
from ..protocol.client import SimulatorClient
|
||||
from ..protocol.log import SessionLog
|
||||
from ..protocol.rules import CLEAR_SUCCESS, DIRECTION, NO_SIGNAL
|
||||
from .features import NUM_ACTIONS, OBS_DIM, action_mask, build_observation, candidate_actions
|
||||
|
||||
|
||||
class LocalizationEnv(gym.Env):
|
||||
"""单源定位清除 Gymnasium 环境。
|
||||
|
||||
problem=3：全向源；problem=4：定向/全向混合（定向概率 0.5，训练分布
|
||||
比题面更稠密以覆盖遮挡分支；正式策略仍由闭环的后备清除兜底）。
|
||||
"""
|
||||
|
||||
metadata = {"render_modes": []}
|
||||
|
||||
def __init__(self, config: ProblemConfig | None = None, *,
|
||||
problem: int = 3, max_episode_steps: int = 16,
|
||||
directional_probability: float = 0.5,
|
||||
real_budget_s: float = 3600.0,
|
||||
error_seed: int | None = None,
|
||||
handoff_probability: float = 0.35, boundary_probability: float = 0.2):
|
||||
super().__init__()
|
||||
if problem not in (3, 4):
|
||||
raise ValueError("problem 必须是 3 或 4")
|
||||
self.config = config or ProblemConfig()
|
||||
self.problem = int(problem)
|
||||
self.max_episode_steps = int(max_episode_steps)
|
||||
self.directional_probability = float(directional_probability)
|
||||
self.real_budget_s = float(real_budget_s)
|
||||
self.error_seed = error_seed
|
||||
self.handoff_probability = handoff_probability
|
||||
self.boundary_probability = boundary_probability
|
||||
self.observation_space = spaces.Box(low=-10.0, high=10.0,
|
||||
shape=(OBS_DIM,), dtype=np.float32)
|
||||
self.action_space = spaces.Discrete(NUM_ACTIONS)
|
||||
|
||||
# 惰性导入：本模块只属于训练/评估侧，运行闭环不加载 referee。
|
||||
from referee import LocalSimulator as _RefereeLocalSimulator
|
||||
from referee import Source
|
||||
from ..harness import LocalSimTransport, referee_spec
|
||||
self._source_cls = Source
|
||||
self._local_simulator_cls = _RefereeLocalSimulator
|
||||
self._local_sim_transport_cls = LocalSimTransport
|
||||
self._referee_spec = referee_spec
|
||||
|
||||
self._scene_rng = Random()
|
||||
self._simulator = None
|
||||
self._client = None
|
||||
self._source = None
|
||||
self._belief: BeliefState | None = None
|
||||
self._channel: int = 1
|
||||
self._blocked: list = []
|
||||
self._directional_observed = False
|
||||
self._clear_failed = False
|
||||
self._steps = 0
|
||||
self._terminated = False
|
||||
self._truncated = False
|
||||
self._measured_points: set = set()
|
||||
self._fallback_attempts = 0
|
||||
|
||||
# ------------------------------------------------------------------ 场景
|
||||
def _make_source_and_probe(self):
|
||||
"""用真值布设一个"首次测向必为 direction"的课程场景（真值不进观测）。"""
|
||||
rng = self._scene_rng
|
||||
field = self.config.field
|
||||
channel = rng.randint(field.channel_min, field.channel_max)
|
||||
radius = field.omega_radius_m * math.sqrt(rng.random())
|
||||
edge = rng.random() < self.boundary_probability
|
||||
if edge:
|
||||
radius = field.omega_radius_m * rng.uniform(0.98, 1.0)
|
||||
angle = rng.uniform(0.0, 2 * math.pi)
|
||||
x, y = radius * math.cos(angle), radius * math.sin(angle)
|
||||
r_eff = rng.uniform(field.r_eff_min_m, field.r_eff_max_m)
|
||||
if edge:
|
||||
r_eff = field.r_eff_min_m
|
||||
directional = self.problem == 4 and rng.random() < self.directional_probability
|
||||
heading = rng.uniform(0.0, 360.0) if directional else 0.0
|
||||
|
||||
# 首次检测点：距离 [300, min(1450, r_eff-20)] 内随机，定向源取朝向
|
||||
# ±89.9°（接近但不跨越90°边界，保证direction）。机器狗可到Ω外 [题面]。
|
||||
max_distance = max(310.0, min(1450.0, r_eff - 20.0))
|
||||
max_distance = max(max_distance, 310.0)
|
||||
probe_distance = rng.uniform(300.0, max_distance)
|
||||
probe_bearing = (heading + rng.uniform(-89.9, 89.9)
|
||||
if directional else rng.uniform(0.0, 360.0))
|
||||
probe_radians = math.radians(probe_bearing)
|
||||
probe = (x + probe_distance * math.cos(probe_radians),
|
||||
y + probe_distance * math.sin(probe_radians))
|
||||
source = self._source_cls(channel=channel, x=x, y=y, r_eff_m=r_eff,
|
||||
directional=directional, heading_deg=heading)
|
||||
return source, probe
|
||||
|
||||
def _make_episode(self) -> bool:
|
||||
"""重建模拟器并完成首次测向；返回 False 表示连续未得到 direction。"""
|
||||
for _ in range(20):
|
||||
self._source, probe = self._make_source_and_probe()
|
||||
self._simulator = self._local_simulator_cls(
|
||||
[self._source], robot_id="rl-train",
|
||||
spec=self._referee_spec(
|
||||
self.config, max_real_duration_s=self.real_budget_s),
|
||||
error_seed=(self.error_seed if self.error_seed is not None
|
||||
else self._scene_rng.randint(0, 2 ** 31 - 1)),
|
||||
stress_mode=True)
|
||||
self._client = SimulatorClient(
|
||||
"rl-train", config=self.config,
|
||||
transport=self._local_sim_transport_cls(self._simulator),
|
||||
log=SessionLog())
|
||||
self._client.enter()
|
||||
outcome = self._client.measure(probe, self._source.channel)
|
||||
if outcome.result == DIRECTION:
|
||||
self._channel = self._source.channel
|
||||
self._belief = BeliefState(self.config)
|
||||
self._belief.update_measure(self._channel, probe, outcome.result,
|
||||
outcome.svd_deg)
|
||||
self._blocked = []
|
||||
self._directional_observed = False
|
||||
self._clear_failed = False
|
||||
self._measured_points = {self._point_key(probe)}
|
||||
self._fallback_attempts = 0
|
||||
# 运行中可能先服务其他频道后再定位本源；模拟这种位置/频道交接。
|
||||
if self._scene_rng.random() < self.handoff_probability:
|
||||
angle = self._scene_rng.uniform(0, 2*math.pi)
|
||||
reach = self.config.field.omega_radius_m * math.sqrt(self._scene_rng.random())
|
||||
other = (self.config.field.channel_min
|
||||
if self._channel != self.config.field.channel_min
|
||||
else self.config.field.channel_max)
|
||||
self._client.measure((reach*math.cos(angle), reach*math.sin(angle)), other)
|
||||
self._start_virtual_s = self._client.state.virtual_time_s
|
||||
return True
|
||||
# near / no_signal 首测属于概率为零的边界或课程构造失败，换场景。
|
||||
continue
|
||||
return False
|
||||
|
||||
# ------------------------------------------------------------------ Gym API
|
||||
@staticmethod
|
||||
def _point_key(point) -> tuple[int, int]:
|
||||
return (int(round(point[0] * 2)), int(round(point[1] * 2)))
|
||||
|
||||
def reset(self, *, seed: int | None = None, options: dict | None = None):
|
||||
super().reset(seed=seed)
|
||||
self.close()
|
||||
if seed is not None:
|
||||
self._scene_rng = Random(seed)
|
||||
self._steps = 0
|
||||
self._localize_steps = 0
|
||||
self._failure_reason = None
|
||||
self._terminated = False
|
||||
self._truncated = False
|
||||
if not self._make_episode():
|
||||
raise RuntimeError("连续 20 次无法构造 direction 首测场景")
|
||||
observation = self._observe()
|
||||
return observation, self._info()
|
||||
|
||||
def _observe(self) -> np.ndarray:
|
||||
region = self._belief.candidate_set(self._channel)
|
||||
if region is None:
|
||||
raise RuntimeError("首次测向后信念状态缺失候选集")
|
||||
return build_observation(
|
||||
region, self._client.state.position, self._channel, self.config,
|
||||
self.problem, self._blocked, self._belief.channels[self._channel],
|
||||
virtual_time_s=self._client.state.virtual_time_s,
|
||||
receiver_channel=self._client.state.channel)
|
||||
|
||||
def action_masks(self):
|
||||
return action_mask(self._belief.candidate_set(self._channel), self._client.state.position,
|
||||
self.config, self.problem, self._blocked,
|
||||
self._belief.channels[self._channel])
|
||||
|
||||
def _info(self) -> dict:
|
||||
source = self._source
|
||||
return {
|
||||
"channel": self._channel,
|
||||
"source": {"x": source.x, "y": source.y, "r_eff_m": source.r_eff_m,
|
||||
"directional": source.directional,
|
||||
"heading_deg": source.heading_deg},
|
||||
"steps": self._steps,
|
||||
"blocked_count": len(self._blocked),
|
||||
"directional_observed": self._directional_observed,
|
||||
"fallback_attempts": self._fallback_attempts,
|
||||
"virtual_cost_s": self._client.state.virtual_time_s - self._start_virtual_s,
|
||||
"failure_reason": self._failure_reason,
|
||||
}
|
||||
|
||||
def step(self, action):
|
||||
if self._terminated or self._truncated:
|
||||
raise RuntimeError("step() 需要在 terminated/truncated 后先 reset()")
|
||||
if not 0 <= int(action) < NUM_ACTIONS:
|
||||
raise ValueError(f"动作 {action} 超出 Discrete({NUM_ACTIONS})")
|
||||
action = int(action)
|
||||
self._steps += 1
|
||||
region = self._belief.candidate_set(self._channel)
|
||||
if region is None:
|
||||
return self._finish(observation=self._observe(), reward=-20.0,
|
||||
terminated=True, truncated=False)
|
||||
|
||||
# 达到清除证书且上次清除失败已被新测量重置：先自动清除（闭环语义）。
|
||||
cert = clear_certificate(region, self._channel, self.config)
|
||||
if cert.certified and not self._clear_failed:
|
||||
return self._try_clear(cert, observation=self._observe())
|
||||
|
||||
old_virtual = self._client.state.virtual_time_s
|
||||
candidates = candidate_actions(region, self._client.state.position,
|
||||
self.config, self._blocked)
|
||||
point = candidates[action]
|
||||
try:
|
||||
outcome = self._client.measure(point, self._channel)
|
||||
self._belief.update_measure(self._channel, point, outcome.result,
|
||||
outcome.svd_deg)
|
||||
except Exception as exc:
|
||||
self._failure_reason = type(exc).__name__ + ': ' + str(exc)
|
||||
return self._finish(observation=self._observe(), reward=-100.0,
|
||||
terminated=True, truncated=False)
|
||||
|
||||
cost_s = self._client.state.virtual_time_s - old_virtual
|
||||
reward = -cost_s / 100.0
|
||||
self._measured_points.add(self._point_key(point))
|
||||
new_region = self._belief.candidate_set(self._channel)
|
||||
if outcome.result == NO_SIGNAL and self.problem == 4:
|
||||
center = region.mec()[0]
|
||||
safe = safe_disk_radius(region, self.config)
|
||||
if safe >= 0 and math.dist(point, center) <= safe + 1e-6:
|
||||
if not self._directional_observed:
|
||||
self._directional_observed = True
|
||||
self._blocked.append(point)
|
||||
else:
|
||||
self._blocked = []
|
||||
self._localize_steps += 1
|
||||
|
||||
self._clear_failed = False
|
||||
cert = clear_certificate(new_region, self._channel, self.config) \
|
||||
if new_region is not None else None
|
||||
observation = self._observe()
|
||||
if cert is not None and cert.certified:
|
||||
return self._try_clear(cert, observation=observation,
|
||||
reward=reward)
|
||||
if self.problem == 4 and (len(self._blocked) >= self.config.search.max_blocked_probes
|
||||
or self._localize_steps >= self.config.search.max_localize_steps):
|
||||
return self._fallback_clear(observation=observation, reward=reward)
|
||||
if self._steps >= self.max_episode_steps:
|
||||
return self._finish(observation=observation, reward=reward - 100.0,
|
||||
terminated=False, truncated=True)
|
||||
return observation, reward, False, False, self._info()
|
||||
|
||||
def _fallback_clear(self, *, observation, reward: float):
|
||||
"""问题4后备清除语义：遮挡梯子用尽后按 20m 网格覆盖逐格 /clear。
|
||||
|
||||
与 DirectionalExtension._fallback_clear 同口径；训练环境借此让策略
|
||||
学到"梯子耗尽的代价"，运行闭环仍由原后备清除函数执行。
|
||||
"""
|
||||
region = self._belief.candidate_set(self._channel)
|
||||
cells = fallback_clear_cells(region.polygon, self.config)
|
||||
self._fallback_attempts = 0
|
||||
if len(cells) > self.config.search.fallback_max_cells:
|
||||
return self._finish(observation=observation, reward=reward - 100.0,
|
||||
terminated=False, truncated=True)
|
||||
attempts = 0
|
||||
while cells:
|
||||
cell = min(cells, key=lambda c: (
|
||||
math.dist(self._client.state.position, c), c))
|
||||
cells.remove(cell)
|
||||
attempts += 1
|
||||
old_virtual = self._client.state.virtual_time_s
|
||||
outcome = self._client.clear(cell, self._channel)
|
||||
reward += -(self._client.state.virtual_time_s - old_virtual) / 100.0
|
||||
if outcome.result == CLEAR_SUCCESS:
|
||||
self._belief.mark_cleared(self._channel)
|
||||
reward += 20.0
|
||||
self._fallback_attempts = attempts
|
||||
return self._finish(observation=observation, reward=reward,
|
||||
terminated=True, truncated=False)
|
||||
self._fallback_attempts = attempts
|
||||
return self._finish(observation=observation, reward=reward - 100.0,
|
||||
terminated=True, truncated=False)
|
||||
|
||||
def _try_clear(self, cert, *, observation, reward: float = 0.0):
|
||||
old_virtual = self._client.state.virtual_time_s
|
||||
outcome = self._client.clear(cert.center, self._channel)
|
||||
cost_s = self._client.state.virtual_time_s - old_virtual
|
||||
reward += -cost_s / 100.0
|
||||
if outcome.result == CLEAR_SUCCESS:
|
||||
self._belief.mark_cleared(self._channel)
|
||||
reward += 20.0
|
||||
return self._finish(observation=observation, reward=reward,
|
||||
terminated=True, truncated=False)
|
||||
self._clear_failed = True
|
||||
reward -= 1.0
|
||||
if self._steps >= self.max_episode_steps:
|
||||
return self._finish(observation=observation, reward=reward,
|
||||
terminated=False, truncated=True)
|
||||
return observation, reward, False, False, self._info()
|
||||
|
||||
def _finish(self, *, observation, reward, terminated, truncated):
|
||||
self._terminated = bool(terminated)
|
||||
self._truncated = bool(truncated)
|
||||
info = self._info()
|
||||
info["cleared"] = (self._belief.channels[self._channel].status.value
|
||||
== "cleared")
|
||||
self.close()
|
||||
return observation, float(reward), self._terminated, self._truncated, info
|
||||
|
||||
def close(self):
|
||||
if (self._client is not None and self._client.state.entered
|
||||
and not self._client.state.exited):
|
||||
self._client.exit()
|
||||
@@ -1,209 +0,0 @@
|
||||
"""RL 状态特征与候选动作生成（问题3/4 共用，运行时与训练环境同源）。
|
||||
|
||||
设计目标：把"下一检测点去哪"这一连续决策转化为固定顺序的离散动作，
|
||||
让 PPO 策略在 GPU 上学习，同时不触碰闭环的保守性证明。候选集合始终由
|
||||
既有几何规则生成（保证接收盘、对称交会、定向遮挡梯子），RL 只负责在
|
||||
这些物理可行候选中排序；证书、覆盖证明与后备清除仍由 planner 层负责。
|
||||
|
||||
观测维度与候选编号一旦训练即成为模型接口，不得随意重排：
|
||||
- 全局特征 41 维（含频道 one-hot 20 维）；
|
||||
- 每个候选 6 维（相对位移、移动耗时、到 MEC 圆心距离、方位、保证接收标记），
|
||||
共 60 × 6 = 360 维；
|
||||
- 总观测 401 维。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
import numpy as np
|
||||
|
||||
from ..config import ProblemConfig, Vec2
|
||||
from ..planner import safe_disk_radius
|
||||
from ._constants import GLOBAL_DIM, LADDER_SLOTS, NUM_ACTIONS, OBS_DIM, PER_ACTION_DIM
|
||||
|
||||
__all__ = [
|
||||
"GLOBAL_DIM",
|
||||
"LADDER_SLOTS",
|
||||
"NUM_ACTIONS",
|
||||
"OBS_DIM",
|
||||
"PER_ACTION_DIM",
|
||||
"candidate_actions",
|
||||
"build_observation",
|
||||
"empty_observation",
|
||||
"action_mask",
|
||||
]
|
||||
|
||||
|
||||
# 候选编号布局（训练/推理共享）：
|
||||
# 0 保证接收盘圆心（或宽候选集 MEC 圆心）
|
||||
# 1..36 保证接收盘 3 环 × 12 方向
|
||||
# 37..51 相对"机器人->MEC 圆心"来向的对称交会候选 3 环 × 5 偏角
|
||||
# 52..59 定向遮挡环探梯子 8 个方向（问题3也保留，作为宽环探索）
|
||||
_RING_RATIOS = (0.4, 0.75, 1.0)
|
||||
_RING_ANGLES = 12
|
||||
_GEO_OFFSETS = (-math.pi / 3, -math.pi / 6, 0.0, math.pi / 6, math.pi / 3)
|
||||
|
||||
|
||||
def _probe_radius(region_radius_m: float, safe_m: float) -> float:
|
||||
"""环候选的基准半径：有保证接收盘用盘半径，否则退化为 MEC 半径。"""
|
||||
if safe_m >= 0.0:
|
||||
return safe_m
|
||||
return max(region_radius_m, 10.0)
|
||||
|
||||
|
||||
def candidate_actions(region, position: Vec2, config: ProblemConfig,
|
||||
blocked: list[Vec2] | None = None) -> list[Vec2]:
|
||||
"""生成固定顺序的 60 个候选检测点。
|
||||
|
||||
参数 region 为 belief.CandidateRegion（任意凸候选集）；blocked 是
|
||||
定向遮挡记录（问题4），问题3 传空列表即可。返回顺序与模型训练时一致。
|
||||
"""
|
||||
center, radius = region.mec()
|
||||
safe = safe_disk_radius(region, config)
|
||||
base_radius = _probe_radius(radius, safe)
|
||||
# 半径恰为0时也保留固定编号，重复位置由action_mask屏蔽。
|
||||
points = [tuple(center)]
|
||||
for fraction in _RING_RATIOS:
|
||||
for k in range(_RING_ANGLES):
|
||||
angle = (k + 0.5) * 2 * math.pi / _RING_ANGLES
|
||||
points.append((center[0] + base_radius*fraction*math.cos(angle),
|
||||
center[1] + base_radius*fraction*math.sin(angle)))
|
||||
assert len(points) == 37
|
||||
|
||||
base_angle = math.atan2(position[1] - center[1],
|
||||
position[0] - center[0])
|
||||
for fraction in _RING_RATIOS:
|
||||
for offset in _GEO_OFFSETS:
|
||||
angle = base_angle + offset
|
||||
points.append((center[0] + base_radius * fraction * math.cos(angle),
|
||||
center[1] + base_radius * fraction * math.sin(angle)))
|
||||
|
||||
blocked = list(blocked or [])
|
||||
first = blocked[0] if blocked else None
|
||||
anchor = 0.0
|
||||
if first is not None:
|
||||
anchor = math.atan2(first[1] - center[1], first[0] - center[0])
|
||||
ladder_radius = max(radius, 20.0)
|
||||
for slot in range(LADDER_SLOTS):
|
||||
angle = anchor + slot * math.pi / 4
|
||||
points.append((center[0] + ladder_radius * math.cos(angle),
|
||||
center[1] + ladder_radius * math.sin(angle)))
|
||||
assert len(points) == NUM_ACTIONS
|
||||
return points
|
||||
|
||||
|
||||
def action_mask(region, position: Vec2, config: ProblemConfig, problem: int,
|
||||
blocked=None, belief=None) -> np.ndarray:
|
||||
"""只依据可见观测屏蔽同地重测、重复候选、越界点。
|
||||
|
||||
Q3有距离保证盘时屏蔽盘外探点；Q4允许盘外探点，但绝不把距离保证
|
||||
当作朝向保证。训练与推理共用此函数，不读取源位置或朝向。
|
||||
"""
|
||||
def key(p):
|
||||
return (round(p[0]*2), round(p[1]*2))
|
||||
visited = {key(r['position']) for r in getattr(belief, 'measurements', [])}
|
||||
seen = set()
|
||||
center, _ = region.mec()
|
||||
safe = safe_disk_radius(region, config)
|
||||
result = []
|
||||
for p in candidate_actions(region, position, config, blocked):
|
||||
tag = key(p)
|
||||
valid = (tag not in visited and tag not in seen
|
||||
and all(math.isfinite(v) and abs(v) <= config.field.coord_abs_limit_m for v in p)
|
||||
and (problem != 3 or safe < 0 or math.dist(p, center) <= safe + 1e-6))
|
||||
result.append(valid)
|
||||
if valid:
|
||||
seen.add(tag)
|
||||
if not any(result):
|
||||
# 退化几何无新候选时保留一个合法动作，由步数/预算边界收尾。
|
||||
result[0] = True
|
||||
return np.asarray(result, dtype=bool)
|
||||
|
||||
|
||||
def _last_measurement_features(belief) -> tuple[tuple[float, float, float],
|
||||
tuple[float, float]]:
|
||||
"""末次读数的 one-hot + 方向角 sin/cos（无读数取零）。"""
|
||||
readings = getattr(belief, "measurements", None) or []
|
||||
if not readings:
|
||||
return (1.0, 0.0, 0.0), (0.0, 0.0)
|
||||
last = readings[-1]
|
||||
result = last.get("result")
|
||||
no_signal = 1.0 if result == "no_signal" else 0.0
|
||||
direction = 1.0 if result == "direction" else 0.0
|
||||
near = 1.0 if result == "near" else 0.0
|
||||
svd = last.get("svd_deg")
|
||||
if direction and svd is not None:
|
||||
radians = math.radians(float(svd))
|
||||
return (no_signal, direction, near), (math.sin(radians), math.cos(radians))
|
||||
return (no_signal, direction, near), (0.0, 0.0)
|
||||
|
||||
|
||||
def build_observation(region, position: Vec2, channel: int, config: ProblemConfig,
|
||||
problem: int, blocked: list[Vec2] | None = None,
|
||||
belief=None, virtual_time_s: float = 0.0,
|
||||
receiver_channel: int | None = None) -> np.ndarray:
|
||||
"""构造 PPO 观测向量（shape=(401,)，与训练环境逐位一致）。"""
|
||||
field, search = config.field, config.search
|
||||
center, radius = region.mec()
|
||||
safe = safe_disk_radius(region, config)
|
||||
readings, svd_pair = _last_measurement_features(belief)
|
||||
xs = [p[0] for p in region.polygon]
|
||||
ys = [p[1] for p in region.polygon]
|
||||
|
||||
global_features = [
|
||||
position[0] / 2000.0,
|
||||
position[1] / 2000.0,
|
||||
((channel if receiver_channel is None else receiver_channel) - field.channel_min)
|
||||
/ max(field.channel_max - field.channel_min, 1),
|
||||
center[0] / 2000.0,
|
||||
center[1] / 2000.0,
|
||||
radius / 2000.0,
|
||||
max(-1.0, min(1.0, safe / 1000.0)),
|
||||
min(len(getattr(belief, "measurements", []) if belief is not None else []),
|
||||
12) / 12.0,
|
||||
min(float(virtual_time_s or 0.0), field.max_virtual_duration_s)
|
||||
/ field.max_virtual_duration_s,
|
||||
# 证书信息由运行时的 clear_certificate 判定传入；训练环境用 region.mec
|
||||
# 的保守近似：radius + slack <= 20 - margin 即可清除。
|
||||
1.0 if radius + region.slack_m <= field.clear_radius_m - search.clear_margin_m else 0.0,
|
||||
float(problem == 4),
|
||||
min(len(blocked or []) / max(search.max_blocked_probes, 1), 1.0),
|
||||
(blocked[0][0] / 2000.0) if blocked else 0.0,
|
||||
(blocked[0][1] / 2000.0) if blocked else 0.0,
|
||||
readings[0],
|
||||
readings[1],
|
||||
readings[2],
|
||||
svd_pair[0],
|
||||
svd_pair[1],
|
||||
(max(xs) - min(xs)) / 4000.0,
|
||||
(max(ys) - min(ys)) / 4000.0,
|
||||
]
|
||||
one_hot = np.zeros(field.channel_max - field.channel_min + 1, dtype=np.float32)
|
||||
one_hot[int(channel) - field.channel_min] = 1.0
|
||||
global_features.extend(one_hot.tolist())
|
||||
assert len(global_features) == GLOBAL_DIM
|
||||
|
||||
action_features: list[float] = []
|
||||
for point in candidate_actions(region, position, config, blocked):
|
||||
dx, dy = point[0] - position[0], point[1] - position[1]
|
||||
distance = math.hypot(dx, dy)
|
||||
action_features.extend([
|
||||
dx / 2000.0,
|
||||
dy / 2000.0,
|
||||
min(distance / (config.cost.speed_mps * 1500.0), 2.0), # 移动耗时 / 1500s
|
||||
math.hypot(point[0] - center[0], point[1] - center[1]) / 2000.0,
|
||||
(math.degrees(math.atan2(dy, dx)) % 360.0) / 360.0,
|
||||
1.0 if safe >= 0 and math.hypot(point[0] - center[0],
|
||||
point[1] - center[1]) <= safe + 1e-6 else 0.0,
|
||||
])
|
||||
assert len(action_features) == NUM_ACTIONS * PER_ACTION_DIM
|
||||
|
||||
observation = np.asarray(global_features + action_features, dtype=np.float32)
|
||||
assert observation.shape == (OBS_DIM,)
|
||||
return observation
|
||||
|
||||
|
||||
def empty_observation() -> np.ndarray:
|
||||
"""依赖缺失时的零观测（仅用于错误提示路径，不进入训练数据）。"""
|
||||
return np.zeros(OBS_DIM, dtype=np.float32)
|
||||
@@ -1,95 +0,0 @@
|
||||
"""RL 选点策略加载与推理（运行时接入 planner 的唯一入口）。
|
||||
|
||||
运行闭环对依赖保持惰性：只有显式选择 ``--strategy rl`` / ``--q4-strategy rl``
|
||||
时才导入 stable_baselines3/torch。模型文件由 scripts/train_rl.py 生成，
|
||||
观测/动作布局与训练环境共享 features 模块。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
import json
|
||||
|
||||
from ..config import ProblemConfig, Vec2
|
||||
from ._constants import NUM_ACTIONS, OBS_DIM
|
||||
from .features import action_mask, build_observation, candidate_actions
|
||||
|
||||
|
||||
class PointSelectionPolicy:
|
||||
"""加载 PPO 模型，把当前候选集映射到 60 个候选检测点之一。"""
|
||||
|
||||
def __init__(self, model_path: str | Path, *, device: str = "auto"):
|
||||
path = Path(model_path)
|
||||
if not path.is_file():
|
||||
raise RuntimeError(
|
||||
f"未找到 RL 模型 {path}。请先安装 rl 依赖组并训练：\n"
|
||||
f" uv sync --group rl --group dev\n"
|
||||
f" uv run python scripts/train_rl.py --problem 3 --timesteps 120000\n"
|
||||
f" uv run python scripts/train_rl.py --problem 4 --timesteps 40000")
|
||||
try:
|
||||
from stable_baselines3 import PPO
|
||||
except ImportError as exc:
|
||||
raise RuntimeError(
|
||||
"缺少 RL 运行依赖（stable-baselines3）。请执行 "
|
||||
"`uv sync --group rl` 后重试。") from exc
|
||||
if device == "auto":
|
||||
try:
|
||||
import torch
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
except ImportError:
|
||||
device = "cpu"
|
||||
self.device = device
|
||||
meta_path = path.with_suffix('.json')
|
||||
self.metadata = (json.loads(meta_path.read_text(encoding='utf-8'))
|
||||
if meta_path.exists() else {})
|
||||
self.feature_version = self.metadata.get('feature_version', 1)
|
||||
if self.feature_version not in (1, 2):
|
||||
raise RuntimeError('不支持的RL特征版本')
|
||||
self.masked = self.metadata.get('algorithm') == 'maskable_ppo'
|
||||
if self.masked:
|
||||
from sb3_contrib import MaskablePPO
|
||||
if self.metadata.get('feature_version') != 2:
|
||||
raise RuntimeError('动作屏蔽模型的特征版本不匹配')
|
||||
loader = MaskablePPO
|
||||
else:
|
||||
loader = PPO
|
||||
self.model = loader.load(str(path), device=device)
|
||||
if not self.masked and type(self.model.policy).__module__.startswith('sb3_contrib'):
|
||||
raise RuntimeError('动作屏蔽模型缺少匹配的JSON元数据，不能按普通PPO推理')
|
||||
try:
|
||||
obs_dim = self.model.observation_space.shape[0]
|
||||
action_dim = self.model.action_space.n
|
||||
except Exception as exc:
|
||||
raise RuntimeError(f"模型空间与 RL 接口不兼容: {path}") from exc
|
||||
if obs_dim != OBS_DIM or action_dim != NUM_ACTIONS:
|
||||
raise RuntimeError(
|
||||
f"模型接口失配: 期望 obs={OBS_DIM}/act={NUM_ACTIONS}，"
|
||||
f"实际 obs={obs_dim}/act={action_dim}。请用当前版本重新训练。")
|
||||
self.path = path
|
||||
|
||||
def select(self, region, position: Vec2, channel: int,
|
||||
config: ProblemConfig, problem: int,
|
||||
blocked: list[Vec2] | None = None, belief=None,
|
||||
virtual_time_s: float = 0.0,
|
||||
receiver_channel: int | None = None) -> tuple[Vec2, int]:
|
||||
"""返回 (检测点, 候选编号)；确定性 argmax（评估/比赛口径）。"""
|
||||
if self.metadata.get('problem', problem) != problem:
|
||||
raise RuntimeError('模型训练问题编号与当前第三/四问不匹配')
|
||||
observation = build_observation(
|
||||
region, position, channel, config, problem, blocked, belief,
|
||||
virtual_time_s=virtual_time_s,
|
||||
receiver_channel=receiver_channel if self.feature_version >= 2 else None)
|
||||
kwargs = ({'action_masks': action_mask(region, position, config, problem,
|
||||
blocked, belief)}
|
||||
if self.masked else {})
|
||||
action, _state = self.model.predict(observation, deterministic=True, **kwargs)
|
||||
action = int(action)
|
||||
candidates = candidate_actions(region, position, config, blocked)
|
||||
return candidates[action], action
|
||||
|
||||
|
||||
def default_model_path(problem: int) -> Path:
|
||||
"""各问题的默认模型位置（训练脚本输出口径）。"""
|
||||
if problem not in (3, 4):
|
||||
raise ValueError("problem 必须是 3 或 4")
|
||||
return Path("output/rl") / f"q{problem}_ppo.zip"
|
||||
@@ -1,15 +0,0 @@
|
||||
"""模型训练前端：本地回环 Web UI + 训练子进程监控。
|
||||
|
||||
启动：
|
||||
python -m mcm26b.training_ui --ui-port 8036
|
||||
或：
|
||||
python scripts/train_hrl_ui.py --ui-port 8036
|
||||
|
||||
子命令：
|
||||
python -m mcm26b.training_ui --print-schema # 配置契约
|
||||
python -m mcm26b.training_ui --print-preview # 离线示例快照
|
||||
"""
|
||||
from .server import TrainingUI, TrainingJob, run_training_ui
|
||||
from .schema import config_schema
|
||||
|
||||
__all__ = ["TrainingUI", "TrainingJob", "run_training_ui", "config_schema"]
|
||||
@@ -1,4 +0,0 @@
|
||||
from .server import run_training_ui
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(run_training_ui())
|
||||
@@ -1,397 +0,0 @@
|
||||
"""训练快照的纯函数派生：曲线统计、收敛判读、进程视图。
|
||||
|
||||
从 ``server.py`` 拆出来的两个理由：
|
||||
|
||||
1. 断开 ``demo`` ↔ ``server`` 的循环导入——离线预览（``demo.py``）只想复用
|
||||
这两个纯函数，没必要反向 import 整个 HTTP 服务器模块；
|
||||
2. 让"数值判读"可以脱离 HTTP 与子进程单独测试。
|
||||
|
||||
本模块只依赖标准库，不做任何 IO、不持有任何状态。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
__all__ = ["MAX_LOG_LINES", "MAX_METRICS", "finite_float",
|
||||
"summarize_convergence", "describe_process", "process_pid"]
|
||||
|
||||
MAX_LOG_LINES = 4000
|
||||
MAX_METRICS = 5000
|
||||
|
||||
_NUM_RE = re.compile(r"^[+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?$")
|
||||
|
||||
|
||||
def finite_float(value: Any) -> float | None:
|
||||
"""把任意输入转成有限浮点数；NaN / ±inf / 非数字一律返回 None。
|
||||
|
||||
旧实现用 ``value != value`` 判 NaN，pylint 报 ``comparison-with-itself``；
|
||||
这里用 ``math.isfinite`` 表达同一语义，意图更直白。
|
||||
"""
|
||||
if value is None or isinstance(value, bool):
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
value = float(value)
|
||||
if not math.isfinite(value):
|
||||
return None
|
||||
return value
|
||||
text = str(value).strip()
|
||||
if not _NUM_RE.match(text):
|
||||
return None
|
||||
try:
|
||||
return float(text)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _series(metrics: list[dict[str, Any]], key: str) -> list[tuple[float, float]]:
|
||||
points: list[tuple[float, float]] = []
|
||||
for point in metrics:
|
||||
value = finite_float(point.get(key))
|
||||
steps = finite_float(point.get("timesteps")) or finite_float(
|
||||
point.get("total_timesteps"))
|
||||
if value is None:
|
||||
continue
|
||||
points.append((steps if steps is not None else float(len(points) + 1), value))
|
||||
return points
|
||||
|
||||
|
||||
def _last_value(points: list[tuple[float, float]]) -> float | None:
|
||||
return points[-1][1] if points else None
|
||||
|
||||
|
||||
def _spread(points: list[tuple[float, float]], window: int = 10
|
||||
) -> tuple[float, float] | None:
|
||||
"""最近 window 个点的 (极差, 起点绝对值)，用于判断"是否还在动"。"""
|
||||
sample = points[-window:]
|
||||
if len(sample) < 3:
|
||||
return None
|
||||
ys = [item[1] for item in sample]
|
||||
return max(ys) - min(ys), abs(ys[0])
|
||||
|
||||
|
||||
def _cv(points: list[tuple[float, float]], window: int = 10) -> float | None:
|
||||
"""最近 window 个点的变异系数（标准差/|均值|），越小越平稳。"""
|
||||
sample = [item[1] for item in points[-window:]]
|
||||
if len(sample) < 3:
|
||||
return None
|
||||
mean = sum(sample) / len(sample)
|
||||
if abs(mean) < 1e-12:
|
||||
return None
|
||||
var = sum((value - mean) ** 2 for value in sample) / len(sample)
|
||||
return (var ** 0.5) / abs(mean)
|
||||
|
||||
|
||||
def _slope(points: list[tuple[float, float]], window: int = 8) -> float | None:
|
||||
sample = points[-window:]
|
||||
if len(sample) < 3:
|
||||
return None
|
||||
xs = [item[0] for item in sample]
|
||||
ys = [item[1] for item in sample]
|
||||
n = len(xs)
|
||||
mean_x = sum(xs) / n
|
||||
mean_y = sum(ys) / n
|
||||
denom = sum((x - mean_x) ** 2 for x in xs)
|
||||
if abs(denom) < 1e-12:
|
||||
return None
|
||||
return sum((x - mean_x) * (y - mean_y) for x, y in zip(xs, ys)) / denom
|
||||
|
||||
|
||||
def _rate_per_1k(reward_points: list[tuple[float, float]],
|
||||
window: int = 8) -> float | None:
|
||||
"""最近 window 个点的奖励增益换算成"每 1000 环境步"，便于读趋势。"""
|
||||
slope = _slope(reward_points, window)
|
||||
if slope is None:
|
||||
return None
|
||||
return slope * 1000.0
|
||||
|
||||
|
||||
def _fmt_num(value: float | None, digits: int = 4) -> str:
|
||||
if value is None:
|
||||
return "—"
|
||||
if value != 0 and abs(value) < 1e-3:
|
||||
return f"{value:.3e}"
|
||||
return f"{value:.{digits}g}"
|
||||
|
||||
|
||||
def _flatness(points: list[tuple[float, float]], window: int = 10
|
||||
) -> tuple[float, float] | None:
|
||||
"""窗口内"趋势位移 / 逐点噪声"比值：<1 表示看不出系统性变化。
|
||||
|
||||
判定收敛时只看斜率绝对值会把"很慢但持续的提升"误判为平台；这里把窗口
|
||||
对半切，用两半均值之差当趋势位移，用相邻点平均绝对差当噪声尺度。返回
|
||||
(比值, 噪声尺度)，两者都小才是真的"又平又稳"。
|
||||
"""
|
||||
sample = points[-window:]
|
||||
if len(sample) < 4:
|
||||
return None
|
||||
ys = [item[1] for item in sample]
|
||||
half = len(ys) // 2
|
||||
shift = abs(sum(ys[half:]) / len(ys[half:]) - sum(ys[:half]) / len(ys[:half]))
|
||||
noise = sum(abs(b - a) for a, b in zip(ys, ys[1:])) / (len(ys) - 1)
|
||||
if noise <= 0:
|
||||
return (0.0, 0.0) if shift <= 0 else (float("inf"), 0.0)
|
||||
return shift / noise, noise
|
||||
|
||||
|
||||
def summarize_convergence(metrics: list[dict[str, Any]],
|
||||
monitor: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
"""给前端返回可解释的收敛状态与各参数最近值。
|
||||
|
||||
这是**启发式判读**，不替代正式学习曲线分析：只依据最近窗口的斜率、
|
||||
极差与变异系数给出"平坦/上升/未上升"的分类，并把同一组数字交给前端
|
||||
展示，便于人工复核。
|
||||
"""
|
||||
reward_points = _series(metrics, "ep_rew_mean")
|
||||
if not reward_points and monitor:
|
||||
reward_points = [(float(index), float(row.get("r", 0.0)))
|
||||
for index, row in enumerate(monitor)
|
||||
if finite_float(row.get("r")) is not None]
|
||||
kl_points = _series(metrics, "approx_kl")
|
||||
ev_points = _series(metrics, "explained_variance")
|
||||
vf_points = _series(metrics, "value_loss")
|
||||
pg_points = _series(metrics, "policy_gradient_loss")
|
||||
entropy_points = _series(metrics, "entropy_loss")
|
||||
fps_points = _series(metrics, "fps")
|
||||
|
||||
reward_slope = _slope(reward_points)
|
||||
reward_rate = _rate_per_1k(reward_points)
|
||||
spread = _spread(reward_points)
|
||||
cv = _cv(reward_points)
|
||||
latest_kl = _last_value(kl_points)
|
||||
latest_ev = _last_value(ev_points)
|
||||
latest_vf = _last_value(vf_points)
|
||||
latest_pg = _last_value(pg_points)
|
||||
latest_entropy = _last_value(entropy_points)
|
||||
latest_fps = _last_value(fps_points)
|
||||
latest_timesteps = metrics[-1].get("timesteps") if metrics else None
|
||||
if latest_timesteps is None and metrics:
|
||||
latest_timesteps = metrics[-1].get("total_timesteps")
|
||||
latest_reward = _last_value(reward_points)
|
||||
latest_len = _last_value(_series(metrics, "ep_len_mean"))
|
||||
latest_updates = _last_value(_series(metrics, "n_updates"))
|
||||
window = min(10, max(3, len(reward_points)))
|
||||
reward_window = [round(value, 6) for _, value in reward_points[-window:]]
|
||||
best_reward = max((value for _, value in reward_points), default=None)
|
||||
|
||||
notes: list[str] = []
|
||||
if not metrics and not monitor:
|
||||
status, tone = "等待首个 rollout", "gray"
|
||||
notes.append("训练进程启动后先输出环境初始化与首个 rollout 的进度行。")
|
||||
elif not metrics:
|
||||
status, tone = "已收到回合奖励，等待进度行", "warn"
|
||||
notes.append("Monitor CSV 已有回合记录，但尚未收到 --progress-json 进度行；"
|
||||
"首个 rollout 结束后会出现完整指标。")
|
||||
elif len(metrics) < 3:
|
||||
status, tone = "预热阶段", "warn"
|
||||
notes.append("已收到初始指标，需要更多 rollout 才能判断趋势。")
|
||||
else:
|
||||
tone, status = "ok", "收敛趋势良好"
|
||||
# 平台判据（三条同时成立）：对半窗口看不出位移、窗口内波动小、
|
||||
# 逐点噪声本身也小。慢速但持续的提升会因第一条被排除。
|
||||
flat = _flatness(reward_points)
|
||||
plateau = False
|
||||
if flat is not None and cv is not None and latest_reward is not None:
|
||||
flat_ratio, flat_noise = flat
|
||||
scale = max(abs(latest_reward), 1e-9)
|
||||
plateau = (flat_ratio <= 1.0 and cv <= 0.15
|
||||
and flat_noise <= 0.02 * scale
|
||||
and (spread is None or spread[1] < 1e-12
|
||||
or spread[0] <= 0.15 * spread[1]))
|
||||
if latest_kl is not None and latest_kl > 0.05:
|
||||
status, tone = "KL 偏大，策略更新可能过猛", "bad"
|
||||
notes.append(f"approx_kl={latest_kl:.4f} > 0.05")
|
||||
elif latest_ev is not None and latest_ev < 0.0:
|
||||
status, tone = "价值函数尚未拟合", "warn"
|
||||
notes.append(f"explained_variance={latest_ev:.3f} < 0")
|
||||
elif reward_slope is not None and reward_slope < 0:
|
||||
status, tone = "奖励未形成上升趋势", "warn"
|
||||
notes.append(f"最近 {window} 点奖励斜率为负（{_fmt_num(reward_slope)}/step）")
|
||||
elif plateau:
|
||||
status, tone = "已趋于平稳（疑似收敛）", "ok"
|
||||
notes.append("窗口内对半位移小于逐点噪声，且波动很小"
|
||||
f"（变异系数 {_fmt_num(cv, 3)}）")
|
||||
else:
|
||||
notes.append(f"最近 {window} 点奖励斜率为正（{_fmt_num(reward_slope)}/step）")
|
||||
if latest_kl is not None and latest_kl <= 0.02:
|
||||
notes.append(f"approx_kl={latest_kl:.4f} 较稳定")
|
||||
if latest_ev is not None:
|
||||
notes.append(f"explained_variance={latest_ev:.3f}")
|
||||
if cv is not None:
|
||||
notes.append(f"奖励变异系数={_fmt_num(cv, 3)}")
|
||||
if latest_entropy is not None and abs(latest_entropy) < 1e-3:
|
||||
notes.append("策略熵接近 0，探索可能过早停止")
|
||||
return {
|
||||
"status": status,
|
||||
"tone": tone,
|
||||
"notes": notes,
|
||||
"window": window,
|
||||
"latest_timesteps": latest_timesteps,
|
||||
"reward_slope": reward_slope,
|
||||
"reward_rate_per_1k": reward_rate,
|
||||
"reward_cv": cv,
|
||||
"reward_spread": spread[0] if spread else None,
|
||||
"reward_window": reward_window,
|
||||
"best_reward": best_reward,
|
||||
"latest_reward": latest_reward,
|
||||
"latest_ep_len": latest_len,
|
||||
"latest_updates": latest_updates,
|
||||
"latest_kl": latest_kl,
|
||||
"latest_explained_variance": latest_ev,
|
||||
"latest_value_loss": latest_vf,
|
||||
"latest_policy_loss": latest_pg,
|
||||
"latest_entropy_loss": latest_entropy,
|
||||
"latest_fps": latest_fps,
|
||||
"reward_points": len(reward_points),
|
||||
}
|
||||
|
||||
|
||||
# 快照状态 -> 界面展示用的进程语义（阶段名称、色调、是否终态）
|
||||
_STATUS_VIEW = {
|
||||
"created": ("已创建", "gray", False, "命令已构造，等待 spawn"),
|
||||
"running": ("运行中", "ok", False, "子进程持有控制台输出与独立进程组"),
|
||||
"paused": ("已暂停", "warn", False, "进程存活但停在 step 边界；点「继续」从暂停处恢复"),
|
||||
"stopping": ("停止中", "warn", False, "已请求优雅停止，等待子进程保存并退出"),
|
||||
"stopped": ("已手动停止", "warn", True, "收到停止请求后退出；脚本会在 step 边界保存模型与元数据"),
|
||||
"exited": ("正常结束", "ok", True, "退出码 0：训练与评估流程走完"),
|
||||
"failed": ("异常结束", "bad", True, "非零退出码或启动失败，见日志尾部"),
|
||||
}
|
||||
|
||||
_PHASES = [
|
||||
("created", "构造命令"),
|
||||
("spawn", "启动子进程"),
|
||||
("rollout", "采集 rollout"),
|
||||
("update", "PPO 更新"),
|
||||
("evaluate", "评估 / 保存"),
|
||||
("done", "收敛归档"),
|
||||
]
|
||||
|
||||
# 状态 -> 当前所处的阶段节点；节点之前为已完成，之后为未开始。
|
||||
_STATUS_PHASE = {
|
||||
"created": "created",
|
||||
"running": "update",
|
||||
"paused": "update",
|
||||
"stopping": "update",
|
||||
"stopped": "done",
|
||||
"exited": "done",
|
||||
"failed": "done",
|
||||
}
|
||||
|
||||
|
||||
def process_pid(process: subprocess.Popen | None) -> int | None:
|
||||
"""已退出进程的 pid 访问会抛 OSError；这里统一吞掉，只用于展示。"""
|
||||
if process is None:
|
||||
return None
|
||||
try:
|
||||
return int(process.pid)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def describe_process(snapshot: dict[str, Any]) -> dict[str, Any]:
|
||||
"""把 snapshot 派生为界面直接可渲染的进程对象。
|
||||
|
||||
包含：状态语义词、进度（步/百分比）、墙钟与剩余时间估计、最近吞吐、
|
||||
进程时间线节点。数字都取自 snapshot，前端不再重复累加，避免两个刷新
|
||||
频率不同的计数器互相打架。
|
||||
"""
|
||||
status = str(snapshot.get("status") or "created")
|
||||
label, tone, terminal, hint = _STATUS_VIEW.get(
|
||||
status, (status, "gray", False, ""))
|
||||
config = snapshot.get("config") or {}
|
||||
order = [key for key, _ in _PHASES]
|
||||
current = _STATUS_PHASE.get(status, "created")
|
||||
timeline = []
|
||||
for key, title in _PHASES:
|
||||
index = order.index(key)
|
||||
if index < order.index(current):
|
||||
state = "done"
|
||||
elif index == order.index(current):
|
||||
state = "active"
|
||||
else:
|
||||
state = "pending"
|
||||
timeline.append({"key": key, "label": title, "state": state})
|
||||
|
||||
metrics = snapshot.get("metrics") or []
|
||||
latest: dict[str, Any] = metrics[-1] if metrics else {}
|
||||
convergence = snapshot.get("convergence") or {}
|
||||
total_steps = finite_float(config.get("timesteps")) or 0.0
|
||||
# SB3 表格行可能没有 steps 字段，因此进度取"所有进度点的最大步数"，
|
||||
# 而不是只看最后一个点——否则末行恰好缺字段时进度会回退到 0。
|
||||
done_steps = 0.0
|
||||
for point in metrics:
|
||||
value = (finite_float(point.get("timesteps"))
|
||||
or finite_float(point.get("total_timesteps")))
|
||||
if value is not None and value > done_steps:
|
||||
done_steps = value
|
||||
if done_steps == 0.0:
|
||||
done_steps = finite_float(convergence.get("latest_timesteps")) or 0.0
|
||||
# init/resume（课程迁移）时 num_timesteps 从上一阶段继续，进度必须按本次
|
||||
# 增量算；start_timesteps 由训练脚本的进度 JSON 提供，缺省按 0 处理。
|
||||
start_steps = 0.0
|
||||
for point in metrics:
|
||||
value = finite_float(point.get("start_timesteps"))
|
||||
if value is not None:
|
||||
start_steps = value
|
||||
break
|
||||
trained_steps = max(0.0, done_steps - start_steps)
|
||||
percent = (100.0 * trained_steps / total_steps) if total_steps > 0 else None
|
||||
if percent is not None:
|
||||
percent = max(0.0, min(100.0, percent))
|
||||
elapsed = float(snapshot.get("elapsed_s") or 0.0)
|
||||
rate = _latest_rate(metrics)
|
||||
remaining: float | None = None
|
||||
eta_ts: float | None = None
|
||||
if rate and rate > 0 and total_steps > 0 and trained_steps < total_steps:
|
||||
remaining = (total_steps - trained_steps) / rate
|
||||
eta_ts = float(snapshot.get("finished_at") or time.time()) + remaining
|
||||
if not terminal and remaining is None:
|
||||
cap = finite_float(config.get("max_wall_s")) or 0.0
|
||||
if cap > 0:
|
||||
remaining = max(0.0, cap - elapsed)
|
||||
return {
|
||||
"label": label,
|
||||
"tone": tone,
|
||||
"terminal": terminal,
|
||||
"hint": hint,
|
||||
"phase": current,
|
||||
"timeline": timeline,
|
||||
"percent": percent,
|
||||
"done_steps": done_steps,
|
||||
"start_steps": start_steps,
|
||||
"trained_steps": trained_steps,
|
||||
"total_steps": total_steps,
|
||||
"elapsed_s": round(elapsed, 3),
|
||||
"rate_per_s": rate,
|
||||
"remaining_s": remaining,
|
||||
"eta_epoch_s": eta_ts,
|
||||
"log_total": snapshot.get("log_total"),
|
||||
"log_capacity": MAX_LOG_LINES,
|
||||
"metric_total": snapshot.get("metric_total"),
|
||||
"latest": latest,
|
||||
}
|
||||
|
||||
|
||||
def _latest_rate(metrics: list[dict[str, Any]], window: int = 6) -> float | None:
|
||||
"""最近若干个**带 fps 的**进度点的平均吞吐（步/秒），用于剩余时间估计。
|
||||
|
||||
progress JSON 行不含 fps，只有 SB3 表格行有；若只看最后 window 个点，
|
||||
恰好落在若干 JSON 行之间时会得到 None，剩余时间就会退化成墙钟上限的
|
||||
粗糙估计。这里向前扫描直到拿到至少 1 个 fps（最多看 3*window 个点）。
|
||||
"""
|
||||
if not metrics:
|
||||
return None
|
||||
usable: list[float] = []
|
||||
for point in reversed(metrics[-window * 3:]):
|
||||
value = finite_float(point.get("fps"))
|
||||
if value and value > 0:
|
||||
usable.append(value)
|
||||
if len(usable) >= window:
|
||||
break
|
||||
if not usable:
|
||||
return None
|
||||
return sum(usable) / len(usable)
|
||||
@@ -1,223 +0,0 @@
|
||||
"""训练前端示例快照：把一段"看起来像真的" PPO 训练曲线合成为合法快照。
|
||||
|
||||
用途有两个，都不涉及真实训练：
|
||||
1. ``GET /api/train/state?preview=1`` 让界面在没有训练进程时也能验证渲染
|
||||
（曲线、指标卡、进程时间线、日志着色、逐回合表）；
|
||||
2. 单元测试可以断言前端的收敛判读分支（上升/平坦/KL 过猛）各有稳定输入。
|
||||
|
||||
曲线是解析函数 + 固定种子的确定性噪声，不含任何真实评测数据。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import random
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from .analysis import describe_process, summarize_convergence
|
||||
|
||||
__all__ = ["DEMO_CONFIG", "DEMO_SCENARIOS", "demo_metrics", "demo_monitor",
|
||||
"demo_log_lines", "demo_snapshot"]
|
||||
|
||||
DEMO_SCENARIOS = ("rising", "plateau", "kl-spike")
|
||||
|
||||
DEMO_CONFIG: dict[str, Any] = {
|
||||
"problem": 3, "stage": 2, "timesteps": 12000, "seed": 320001,
|
||||
"envs": 4, "vec_env": "subproc", "device": "mps",
|
||||
"config_profile": "production", "reward_profile": "time",
|
||||
"model": "output/hrl/ui/demo/q3.zip", "init": None, "resume": False,
|
||||
"eval_episodes": 2, "rollout_steps": 256, "batch_size": 256,
|
||||
"n_epochs": 10, "max_wall_s": 1200.0, "local_selector": "geometry",
|
||||
"local_model": None, "verbose": 1,
|
||||
}
|
||||
|
||||
DEMO_COMMAND = [
|
||||
"python", "scripts/train_hrl_high.py", "--problem", "3", "--stage", "2",
|
||||
"--timesteps", "12000", "--seed", "320001", "--envs", "4",
|
||||
"--vec-env", "subproc", "--device", "mps", "--config-profile", "production",
|
||||
"--reward-profile", "time", "--model", "output/hrl/ui/demo/q3.zip",
|
||||
"--eval-episodes", "2", "--rollout-steps", "256", "--batch-size", "256",
|
||||
"--n-epochs", "10", "--max-wall-s", "1200.0", "--local-selector",
|
||||
"geometry", "--progress-json", "--verbose", "1",
|
||||
]
|
||||
|
||||
DEMO_STEPS = 48
|
||||
|
||||
|
||||
def _finite(value: float) -> float | None:
|
||||
if not math.isfinite(value):
|
||||
return None
|
||||
return round(value, 8)
|
||||
|
||||
|
||||
def demo_metrics(count: int = DEMO_STEPS, steps: int = 12000, *,
|
||||
seed: int = 7, scenario: str = "rising",
|
||||
total_steps: int | None = None, fps: float = 420.0
|
||||
) -> list[dict[str, Any]]:
|
||||
"""合成 count 个进度点，横轴覆盖到 ``total_steps`` 的 ``count/DEMO_STEPS``。
|
||||
|
||||
场景（用于验证收敛判读的各个分支，不含真实数据）：
|
||||
rising 奖励对数饱和上升、KL 正常 → "收敛趋势良好"
|
||||
plateau 奖励已进入平台期 → "已趋于平稳（疑似收敛）"
|
||||
kl-spike 末段近似 KL 冲高 → "KL 偏大，策略更新可能过猛"
|
||||
"""
|
||||
if scenario not in DEMO_SCENARIOS:
|
||||
raise ValueError(f"未知示例场景: {scenario}")
|
||||
total = int(total_steps if total_steps is not None else steps)
|
||||
rng = random.Random(seed)
|
||||
points: list[dict[str, Any]] = []
|
||||
per = total / DEMO_STEPS
|
||||
ceiling = 42.0
|
||||
span = min(1.0, max(1e-6, count / float(DEMO_STEPS)))
|
||||
for index in range(1, count + 1):
|
||||
t = (index / float(DEMO_STEPS)) / span # 折到整段训练的进度比例
|
||||
steps_now = int(round(index * per))
|
||||
if scenario == "plateau":
|
||||
reward = 41.5 + rng.uniform(-0.35, 0.35)
|
||||
value_loss_curve = 3.6 + rng.uniform(-0.2, 0.2)
|
||||
approx_kl = max(0.001, 0.004 + rng.uniform(-0.001, 0.001))
|
||||
explained = min(0.95, 0.88 + rng.uniform(-0.02, 0.02))
|
||||
elif scenario == "kl-spike":
|
||||
reward = ceiling * (1.0 - math.exp(-3.1 * t)) + rng.uniform(-1.2, 1.2)
|
||||
value_loss_curve = 62.0 * math.exp(-2.4 * t) + 3.4
|
||||
base_kl = 0.031 * math.exp(-1.7 * t)
|
||||
approx_kl = base_kl + (0.075 if t > 0.8 else 0.0) + rng.uniform(-0.003, 0.003)
|
||||
explained = -0.09 + 1.05 * (1.0 - math.exp(-2.2 * t))
|
||||
else:
|
||||
noise = rng.uniform(-1.4, 1.4) * (1.0 - 0.6 * t)
|
||||
reward = ceiling * (1.0 - math.exp(-3.1 * t)) + noise
|
||||
value_loss_curve = 62.0 * math.exp(-2.4 * t) + 3.4
|
||||
approx_kl = 0.031 * math.exp(-1.7 * t)
|
||||
explained = -0.09 + 1.05 * (1.0 - math.exp(-2.2 * t))
|
||||
value_loss = value_loss_curve + rng.uniform(-0.5, 0.5)
|
||||
policy_loss = -0.021 + 0.006 * math.sin(index / 4.0) + rng.uniform(-0.002, 0.002)
|
||||
entropy = 1.35 - 0.55 * (t if scenario != "plateau" else 0.9) \
|
||||
+ rng.uniform(-0.02, 0.02)
|
||||
points.append({
|
||||
"timesteps": steps_now,
|
||||
"n_updates": index * 12,
|
||||
"ep_rew_mean": _finite(reward),
|
||||
"ep_len_mean": _finite(24.0 + 21.0 * t + rng.uniform(-1.5, 1.5)),
|
||||
"value_loss": _finite(value_loss),
|
||||
"policy_gradient_loss": _finite(policy_loss),
|
||||
"approx_kl": _finite(max(0.0008, approx_kl + rng.uniform(-0.002, 0.002))),
|
||||
"entropy_loss": _finite(entropy),
|
||||
"explained_variance": _finite(min(0.97, explained
|
||||
+ rng.uniform(-0.03, 0.03))),
|
||||
# 吞吐尽量接近真实量级（envs=4 时约 400+ 步/秒），否则剩余时间
|
||||
# 估计会给出荒谬的 ETA
|
||||
"fps": _finite(max(30.0, fps * (0.92 + 0.16 * t)
|
||||
+ rng.uniform(-12.0, 12.0))),
|
||||
})
|
||||
return points
|
||||
|
||||
|
||||
def demo_monitor(count: int = 60, seed: int = 11) -> list[dict[str, Any]]:
|
||||
rng = random.Random(seed)
|
||||
rows = []
|
||||
clock = 0.0
|
||||
for index in range(count):
|
||||
t = index / max(1, count - 1)
|
||||
clock += rng.uniform(4.0, 14.0)
|
||||
rows.append({
|
||||
"r": _finite(38.0 * (1.0 - math.exp(-3.0 * t)) + rng.uniform(-3.0, 3.0)),
|
||||
"l": float(rng.randint(28, 74)),
|
||||
"t": round(clock, 4),
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def demo_log_lines() -> list[str]:
|
||||
"""一段包含各类着色分支的日志：SB3 表格、进度 JSON、警告与收尾。"""
|
||||
lines = [
|
||||
"=== 训练前端离线预览：以下为示例快照，非真实训练输出 ===",
|
||||
"使用设备: mps",
|
||||
"创建 4 个 DummyVecEnv/SubprocVecEnv 环境 (stage=2, problem=3)",
|
||||
"Monitor CSV: output/hrl/ui/demo/high_curve_q3_env0.monitor.csv",
|
||||
"-----------------------------",
|
||||
"| rollout/ | |",
|
||||
"| ep_len_mean | 31.4 |",
|
||||
"| ep_rew_mean | 12.83 |",
|
||||
"| time/ | |",
|
||||
"| fps | 412 |",
|
||||
"| total_timesteps | 512 |",
|
||||
"-----------------------------",
|
||||
]
|
||||
for index in range(1, 9):
|
||||
t = index / 8.0
|
||||
lines.append(
|
||||
'{"event": "progress", "timesteps": %d, "n_updates": %d, '
|
||||
'"ep_rew_mean": %.3f, "ep_len_mean": %.2f, "value_loss": %.4f, '
|
||||
'"policy_gradient_loss": %.5f, "approx_kl": %.5f, '
|
||||
'"entropy_loss": %.4f, "explained_variance": %.4f, "fps": %.1f}'
|
||||
% (index * 1500, index * 12, 42.0 * (1 - math.exp(-3.1 * t)),
|
||||
24 + 21 * t, 62.0 * math.exp(-2.4 * t) + 3.4, -0.021 + 0.006 * t,
|
||||
0.031 * math.exp(-1.7 * t), 1.35 - 0.55 * t,
|
||||
-0.09 + 1.05 * (1 - math.exp(-2.2 * t)), 390 + 70 * t)
|
||||
)
|
||||
lines += [
|
||||
"UserWarning: 示例警告行，用于验证警告着色",
|
||||
"确定性评估：2/2 成功，accounted=20，无异常",
|
||||
"模型已保存: output/hrl/ui/demo/q3.zip",
|
||||
"元数据已写入: output/hrl/ui/demo/q3.json",
|
||||
]
|
||||
return lines
|
||||
|
||||
|
||||
def demo_snapshot(*, log_tail: int = 400, scenario: str = "rising",
|
||||
done_ratio: float = 0.62, fps: float = 420.0) -> dict[str, Any]:
|
||||
"""合成一次"跑到中途"的快照（键与 TrainingJob.snapshot 一致）。
|
||||
|
||||
``done_ratio`` 控制已完成的步数比例，用来让预览更接近真实运行中状态
|
||||
（而不是刚好 100%）；``scenario`` 选择收敛判读分支；``fps`` 决定剩余
|
||||
时间估计的吞吐量级。
|
||||
"""
|
||||
now = time.time()
|
||||
ratio = min(1.0, max(0.05, float(done_ratio)))
|
||||
metrics = demo_metrics(count=max(6, int(round(DEMO_STEPS * ratio))),
|
||||
total_steps=int(DEMO_CONFIG["timesteps"]),
|
||||
scenario=scenario, fps=fps)
|
||||
monitor = demo_monitor(count=max(6, int(round(60 * ratio))))
|
||||
config = dict(DEMO_CONFIG)
|
||||
lines = demo_log_lines()
|
||||
logs = (lines * 12)[-max(1, int(log_tail)):]
|
||||
status = "running"
|
||||
payload: dict[str, Any] = {
|
||||
"job_id": "job-preview",
|
||||
"status": status,
|
||||
"pid": 42424,
|
||||
"created_at": now - 186.0,
|
||||
"started_at": now - 184.0,
|
||||
"finished_at": None,
|
||||
"elapsed_s": 184.0,
|
||||
"returncode": None,
|
||||
"error": None,
|
||||
"forced_kill": False,
|
||||
"command": list(DEMO_COMMAND),
|
||||
"model": str(config["model"]),
|
||||
"config": config,
|
||||
"log_lines": logs,
|
||||
"log_total": len(logs) * 3,
|
||||
"metrics": metrics,
|
||||
"metric_offset": 0,
|
||||
"metric_end": len(metrics),
|
||||
"metric_total": len(metrics),
|
||||
"monitor": monitor,
|
||||
"recovery": {
|
||||
"can_resume": True,
|
||||
"model": str(config["model"]),
|
||||
"timesteps": int(DEMO_STEPS * ratio * 12),
|
||||
"reason": "recovery",
|
||||
"updated_unix_s": now - 24.0,
|
||||
"source": "resume.json",
|
||||
"hint_file": str(config["model"]).rsplit("/", 1)[0] + "/resume.json",
|
||||
"candidates": 3,
|
||||
"preview": True,
|
||||
},
|
||||
"preview": True,
|
||||
"preview_scenario": scenario,
|
||||
}
|
||||
payload["convergence"] = summarize_convergence(metrics, monitor)
|
||||
payload["process"] = describe_process(payload)
|
||||
return payload
|
||||
@@ -1,328 +0,0 @@
|
||||
"""训练前端配置契约：字段元数据、分组、预设与指标字典。
|
||||
|
||||
为什么单独成模块：前端表单的取值域、单位、分组与后端 ``TrainConfig`` 的
|
||||
校验规则必须来自同一处，否则界面会允许后端拒绝的输入（或反之）。这里只
|
||||
声明"可展示"的元数据，真正的合法性判定仍由 :class:`TrainConfig.validate`
|
||||
单点负责——前端只做提前提示，不复制规则。
|
||||
|
||||
指标字典 ``METRIC_CATALOG`` 同样是唯一出处：``server.summarize_convergence``
|
||||
用它取数，``/api/train/schema`` 把它交给前端画图，避免两边各写一份键名。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
__all__ = [
|
||||
"CONFIG_FIELDS", "CONFIG_GROUPS", "PRESETS", "METRIC_CATALOG",
|
||||
"config_schema", "preset_payloads", "metric_keys",
|
||||
]
|
||||
|
||||
# 字段元数据：
|
||||
# kind number|integer|text|select|bool
|
||||
# 其余键 前端直接使用（label/unit/group/choices/min/max/step/help/advanced）
|
||||
CONFIG_FIELDS: list[dict[str, Any]] = [
|
||||
# ---------------------------------------------------------- 环境与课程
|
||||
{
|
||||
"key": "problem", "kind": "select", "group": "env",
|
||||
"label": "问题", "default": 3, "choices": [
|
||||
{"value": 3, "label": "问题 3（全向源）"},
|
||||
{"value": 4, "label": "问题 4（含定向源）"},
|
||||
],
|
||||
"help": "决定场景源类型与末段课程配置。",
|
||||
},
|
||||
{
|
||||
"key": "stage", "kind": "select", "group": "env",
|
||||
"label": "课程阶段", "default": 1, "choices": [
|
||||
{"value": 1, "label": "1 · 单源 / 5 频道 / 无噪"},
|
||||
{"value": 2, "label": "2 · 3 源 / 10 频道 / ±0.5°"},
|
||||
{"value": 3, "label": "3 · 8 源 / 16 频道 / ±1°"},
|
||||
{"value": 4, "label": "4 · 10–16 源 / 20 频道 / ±1°"},
|
||||
],
|
||||
"help": "课程递进；跨阶段续训用下方 --init 迁移权重。",
|
||||
},
|
||||
{
|
||||
"key": "config_profile", "kind": "select", "group": "env",
|
||||
"label": "配置档", "default": "production", "choices": [
|
||||
{"value": "production", "label": "production（生产保真）"},
|
||||
{"value": "coarse-pretrain", "label": "coarse-pretrain（快速粗训）"},
|
||||
],
|
||||
"help": "coarse-pretrain 显式映射为 fast 评分配置并写入模型元数据。",
|
||||
},
|
||||
{
|
||||
"key": "reward_profile", "kind": "select", "group": "env",
|
||||
"label": "奖励口径", "default": "time", "choices": [
|
||||
{"value": "time", "label": "time（P0.4 纯时间目标）"},
|
||||
{"value": "prompt", "label": "prompt（§5.2 原始塑形）"},
|
||||
],
|
||||
"help": "对照实验用；口径固化进模型元数据，不静默替换。",
|
||||
},
|
||||
{
|
||||
"key": "seed", "kind": "integer", "group": "env",
|
||||
"label": "随机种子", "default": 320001, "min": 0, "max": 2_147_483_647,
|
||||
"help": "不同种子用于多次重复训练与方差估计。",
|
||||
},
|
||||
# ---------------------------------------------------------- PPO 超参
|
||||
{
|
||||
"key": "rollout_steps", "kind": "integer", "group": "ppo",
|
||||
"label": "采样步数 n_steps", "default": 128, "min": 1, "max": 100_000,
|
||||
"help": "每次更新前每个环境采集的步数；近似一个 rollout 的宽度。",
|
||||
},
|
||||
{
|
||||
"key": "batch_size", "kind": "integer", "group": "ppo",
|
||||
"label": "批大小 batch_size", "default": 128, "min": 1, "max": 100_000,
|
||||
"help": "建议 batch_size ≤ n_steps × envs。",
|
||||
},
|
||||
{
|
||||
"key": "n_epochs", "kind": "integer", "group": "ppo",
|
||||
"label": "更新轮数 n_epochs", "default": 3, "min": 1, "max": 100,
|
||||
"help": "每批数据的 SGD 轮数；过大易使 KL 飙升。",
|
||||
},
|
||||
# ---------------------------------------------------------- 运行环境
|
||||
{
|
||||
"key": "envs", "kind": "integer", "group": "runtime",
|
||||
"label": "并行环境数", "default": 1, "min": 1, "max": 32,
|
||||
"help": "SubprocVecEnv 下每个 worker 一个进程与 Monitor 文件。",
|
||||
},
|
||||
{
|
||||
"key": "vec_env", "kind": "select", "group": "runtime",
|
||||
"label": "向量化方式", "default": "dummy", "choices": [
|
||||
{"value": "dummy", "label": "dummy（单进程，调试友好）"},
|
||||
{"value": "subproc", "label": "subproc（多进程，吞吐优先）"},
|
||||
],
|
||||
"help": "envs>1 且追求吞吐时选 subproc。",
|
||||
},
|
||||
{
|
||||
"key": "device", "kind": "select", "group": "runtime",
|
||||
"label": "计算设备", "default": "auto", "choices": [
|
||||
{"value": "auto", "label": "auto（自动选择）"},
|
||||
{"value": "cpu", "label": "cpu"},
|
||||
{"value": "cuda", "label": "cuda（NVIDIA）"},
|
||||
{"value": "mps", "label": "mps（Apple 芯片）"},
|
||||
],
|
||||
"help": "CPU 路径最稳；GPU 需与 torch 构建匹配。",
|
||||
},
|
||||
{
|
||||
"key": "local_selector", "kind": "select", "group": "runtime",
|
||||
"label": "低层执行器", "default": "geometry", "choices": [
|
||||
{"value": "geometry", "label": "geometry（几何选点，默认）"},
|
||||
{"value": "rl", "label": "rl（低层 PPO，接口预留）"},
|
||||
],
|
||||
"help": "rl 仅记录元数据，当前 HTTP 执行路径仍由高层宏动作终点决定。",
|
||||
},
|
||||
# ---------------------------------------------------------- 训练规模
|
||||
{
|
||||
"key": "timesteps", "kind": "integer", "group": "scale",
|
||||
"label": "总步数 timesteps", "default": 2048, "min": 1,
|
||||
"max": 10_000_000, "help": "本次训练的环境步总数（续训时为增量步数）。",
|
||||
},
|
||||
{
|
||||
"key": "max_wall_s", "kind": "number", "group": "scale",
|
||||
"label": "墙钟上限 (s)", "default": 1200.0, "min": 0.1, "max": 7200.0,
|
||||
"step": 30, "help": "硬上限：到点由回调主动停止并保存模型。",
|
||||
},
|
||||
{
|
||||
"key": "eval_episodes", "kind": "integer", "group": "scale",
|
||||
"label": "评估回合数", "default": 2, "min": 0, "max": 1000,
|
||||
"help": "训练结束后确定性评估；0 表示跳过。",
|
||||
},
|
||||
{
|
||||
"key": "verbose", "kind": "select", "group": "scale",
|
||||
"label": "SB3 日志级别", "default": 1, "choices": [
|
||||
{"value": 0, "label": "0 · 静默（只留 JSON 进度）"},
|
||||
{"value": 1, "label": "1 · 表格（默认）"},
|
||||
{"value": 2, "label": "2 · 调试"},
|
||||
],
|
||||
"help": "前端已解析 SB3 表格，默认 1 便于对照原始输出。",
|
||||
},
|
||||
# ---------------------------------------------------------- 模型路径
|
||||
{
|
||||
"key": "model", "kind": "text", "group": "paths",
|
||||
"label": "模型输出路径", "default": "output/hrl/ui/q3.zip",
|
||||
"help": "相对仓库根；同目录会写入 .json 元数据与 Monitor CSV。",
|
||||
},
|
||||
{
|
||||
"key": "init", "kind": "text", "group": "paths", "optional": True,
|
||||
"label": "初始权重 --init", "default": None,
|
||||
"help": "留空=从零；填上一阶段 .zip 即课程迁移。PPO 超参与步数计数沿用"
|
||||
"该模型（timesteps 按增量理解，进度条按增量显示）。",
|
||||
},
|
||||
{
|
||||
"key": "local_model", "kind": "text", "group": "paths", "optional": True,
|
||||
"label": "低层模型 --local-model", "default": None,
|
||||
"advanced": True,
|
||||
"help": "仅 local_selector=rl 时有意义。",
|
||||
},
|
||||
{
|
||||
"key": "resume", "kind": "bool", "group": "paths",
|
||||
"label": "续训 --resume", "default": False,
|
||||
"help": "从模型输出路径已有 .zip 继续，timesteps 按增量理解。",
|
||||
},
|
||||
{
|
||||
"key": "force", "kind": "bool", "group": "paths",
|
||||
"label": "覆盖已有模型 --force", "default": False,
|
||||
"help": "输出路径已存在时必须勾选，否则训练脚本会拒绝覆盖。与续训互斥。",
|
||||
},
|
||||
# ---------------------------------------------------------- 恢复与落盘
|
||||
{
|
||||
"key": "recovery_every", "kind": "integer", "group": "recovery",
|
||||
"label": "滚动恢复点间隔 --recovery-every", "default": 2000,
|
||||
"help": "每 N 个环境步原子覆盖写 <checkpoints>/<stem>_latest.zip，"
|
||||
"训练开始前先写起步基线；急停（强杀）后最多丢 N 步。0=关闭。",
|
||||
},
|
||||
{
|
||||
"key": "checkpoint_every", "kind": "integer", "group": "recovery",
|
||||
"label": "历史 checkpoint 间隔 --checkpoint-every", "default": 20000,
|
||||
"help": "每 N 个环境步另留一份 <stem>_<步数>.zip 供返祖对比；"
|
||||
"0=只保留滚动恢复点。",
|
||||
},
|
||||
{
|
||||
"key": "checkpoint_keep", "kind": "integer", "group": "recovery",
|
||||
"label": "历史 checkpoint 保留数 --checkpoint-keep", "default": 3,
|
||||
"help": "只保留最近 K 个历史 checkpoint；滚动恢复点与起步基线不受影响。",
|
||||
},
|
||||
]
|
||||
|
||||
CONFIG_GROUPS: list[dict[str, str]] = [
|
||||
{"key": "env", "label": "环境与课程", "hint": "题目、课程阶段与奖励口径"},
|
||||
{"key": "ppo", "label": "PPO 超参数", "hint": "采样宽度与更新强度"},
|
||||
{"key": "runtime", "label": "运行环境", "hint": "并行、设备与低层执行器"},
|
||||
{"key": "scale", "label": "训练规模", "hint": "步数、墙钟上限与评估"},
|
||||
{"key": "recovery", "label": "恢复与落盘", "hint": "急停恢复点与历史 checkpoint"},
|
||||
{"key": "paths", "label": "模型与迁移", "hint": "产物路径、续训与课程迁移"},
|
||||
]
|
||||
|
||||
# 预设 = 只覆盖差异字段；后端 from_payload 会补齐其余默认值。
|
||||
PRESETS: list[dict[str, Any]] = [
|
||||
{
|
||||
"key": "smoke",
|
||||
"label": "冒烟（约 1 分钟）",
|
||||
"note": "64 步、CPU、无评估——只验证链路与界面。",
|
||||
"payload": {
|
||||
"problem": 3, "stage": 1, "timesteps": 64, "seed": 1,
|
||||
"envs": 1, "vec_env": "dummy", "device": "cpu",
|
||||
"config_profile": "coarse-pretrain", "reward_profile": "time",
|
||||
"model": "output/hrl/ui/smoke/q3.zip", "eval_episodes": 0,
|
||||
"rollout_steps": 32, "batch_size": 32, "n_epochs": 2,
|
||||
"max_wall_s": 180.0, "verbose": 1, "force": True,
|
||||
},
|
||||
},
|
||||
{
|
||||
"key": "local-check",
|
||||
"label": "本地验证（约 10 分钟）",
|
||||
"note": "stage 1 单源课程 2048 步 + 2 回合评估。",
|
||||
"payload": {
|
||||
"problem": 3, "stage": 1, "timesteps": 2048, "seed": 320001,
|
||||
"envs": 1, "vec_env": "dummy", "device": "auto",
|
||||
"config_profile": "production", "reward_profile": "time",
|
||||
"model": "output/hrl/ui/local/q3.zip", "eval_episodes": 2,
|
||||
"rollout_steps": 128, "batch_size": 128, "n_epochs": 3,
|
||||
"max_wall_s": 900.0, "verbose": 1, "force": True,
|
||||
},
|
||||
},
|
||||
{
|
||||
"key": "q3-main",
|
||||
"label": "Q3 主训练",
|
||||
"note": "stage 3 八源课程，4 进程并行；建议在训练机上跑。",
|
||||
"payload": {
|
||||
"problem": 3, "stage": 3, "timesteps": 200000, "seed": 320001,
|
||||
"envs": 4, "vec_env": "subproc", "device": "auto",
|
||||
"config_profile": "production", "reward_profile": "time",
|
||||
"model": "output/hrl/ui/q3-main/q3.zip", "eval_episodes": 5,
|
||||
"rollout_steps": 256, "batch_size": 256, "n_epochs": 10,
|
||||
"max_wall_s": 7200.0, "verbose": 1,
|
||||
},
|
||||
},
|
||||
{
|
||||
"key": "q4-transfer",
|
||||
"label": "Q4 课程迁移",
|
||||
"note": "从 Q3 权重迁移到问题 4 末段课程（含定向源）。",
|
||||
"payload": {
|
||||
"problem": 4, "stage": 4, "timesteps": 120000, "seed": 420001,
|
||||
"envs": 4, "vec_env": "subproc", "device": "auto",
|
||||
"config_profile": "production", "reward_profile": "time",
|
||||
"model": "output/hrl/ui/q4-main/q4.zip",
|
||||
"init": "output/hrl/ui/q3-main/q3.zip", "eval_episodes": 5,
|
||||
"rollout_steps": 256, "batch_size": 256, "n_epochs": 10,
|
||||
"max_wall_s": 7200.0, "verbose": 1,
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
# 指标字典：键来自训练脚本 --progress-json 的单行 JSON（短名）与 SB3 表格
|
||||
# （归一化去前缀）。前端按此顺序绘图，收敛判据在 server.summarize_convergence。
|
||||
METRIC_CATALOG: list[dict[str, Any]] = [
|
||||
{
|
||||
"key": "ep_rew_mean", "label": "平均回合奖励", "role": "primary",
|
||||
"color": "#4f8cff", "format": "number", "better": "up",
|
||||
"help": "最近若干回合的平均回报；主要收敛判据。",
|
||||
},
|
||||
{
|
||||
"key": "ep_len_mean", "label": "平均回合长度", "role": "secondary",
|
||||
"color": "#38bdf8", "format": "number", "better": "neutral",
|
||||
"help": "回合步数；与奖励同时上升通常代表策略更完整而非更快。",
|
||||
},
|
||||
{
|
||||
"key": "approx_kl", "label": "近似 KL", "role": "stability",
|
||||
"color": "#f59e0b", "format": "sci", "better": "down",
|
||||
"help": "策略更新幅度；> 0.05 视为更新过猛。",
|
||||
},
|
||||
{
|
||||
"key": "explained_variance", "label": "解释方差", "role": "stability",
|
||||
"color": "#34d399", "format": "number", "better": "up",
|
||||
"help": "价值函数拟合优度；< 0 表示还不如均值预测。",
|
||||
},
|
||||
{
|
||||
"key": "value_loss", "label": "价值损失", "role": "stability",
|
||||
"color": "#f87171", "format": "sci", "better": "down",
|
||||
"help": "critic 拟合误差。",
|
||||
},
|
||||
{
|
||||
"key": "policy_gradient_loss", "label": "策略梯度损失", "role": "stability",
|
||||
"color": "#c084fc", "format": "sci", "better": "neutral",
|
||||
"help": "actor 的代理损失；量级小且符号可变属正常。",
|
||||
},
|
||||
{
|
||||
"key": "entropy_loss", "label": "策略熵", "role": "exploration",
|
||||
"color": "#fbbf24", "format": "number", "better": "neutral",
|
||||
"help": "探索度；若长期贴近 0 说明策略过早确定。",
|
||||
},
|
||||
{
|
||||
"key": "n_updates", "label": "更新次数", "role": "throughput",
|
||||
"color": "#93a1bd", "format": "integer", "better": "up",
|
||||
"help": "已完成的 PPO 更新轮数。",
|
||||
},
|
||||
{
|
||||
"key": "fps", "label": "吞吐 FPS", "role": "throughput",
|
||||
"color": "#22d3ee", "format": "number", "better": "up",
|
||||
"help": "每秒环境步数，反映硬件与并行配置是否吃满。",
|
||||
},
|
||||
]
|
||||
|
||||
_METRIC_BY_KEY = {item["key"]: item for item in METRIC_CATALOG}
|
||||
|
||||
|
||||
def metric_keys() -> list[str]:
|
||||
return [item["key"] for item in METRIC_CATALOG]
|
||||
|
||||
|
||||
def metric_meta(key: str) -> dict[str, Any]:
|
||||
return _METRIC_BY_KEY.get(key, {"key": key, "label": key, "role": "other",
|
||||
"color": "#93a1bd", "format": "number",
|
||||
"better": "neutral", "help": ""})
|
||||
|
||||
|
||||
def config_schema() -> dict[str, Any]:
|
||||
"""返回给前端的完整配置契约（字段 + 分组 + 预设 + 指标字典）。"""
|
||||
return {
|
||||
"fields": CONFIG_FIELDS,
|
||||
"groups": CONFIG_GROUPS,
|
||||
"presets": [{"key": item["key"], "label": item["label"],
|
||||
"note": item["note"], "payload": dict(item["payload"])}
|
||||
for item in PRESETS],
|
||||
"metrics": METRIC_CATALOG,
|
||||
}
|
||||
|
||||
|
||||
def preset_payloads() -> dict[str, dict[str, Any]]:
|
||||
return {item["key"]: dict(item["payload"]) for item in PRESETS}
|
||||
@@ -1,977 +0,0 @@
|
||||
"""训练前端服务器：在评测前端同一套「本地回环 HTTP + 静态白名单」模式上，
|
||||
额外托管一个受控的训练子进程，并把 stdout/JSON 进度流做成轮询 API。
|
||||
|
||||
设计边界：
|
||||
- 只监听 127.0.0.1，只允许白名单静态文件；
|
||||
- 只允许一个训练任务，命令由服务端用参数列表构造，不经过 shell；
|
||||
- 支持启动/暂停/继续/停止/状态查询；暂停与停止通过**控制文件**下发，
|
||||
POSIX 与 Windows 行为一致（详见 :mod:`mcm26b.process_control`）；
|
||||
- 训练进程通过 ``--progress-json`` 输出单行 JSON 进度，服务端同时兼容
|
||||
Stable-Baselines3 的表格输出和 Monitor CSV。
|
||||
|
||||
界面契约（供 ``static/`` 使用）：
|
||||
- ``GET /api/train/schema`` 字段/分组/预设/指标字典（``schema.py`` 单一出处）
|
||||
- ``GET /api/train/state`` 快照；``after_metric=N`` 只回增量指标
|
||||
- ``POST /api/train/start`` 以部分字段启动（其余取默认值）
|
||||
- ``POST /api/train/pause`` 请求训练进程在 step 边界挂起
|
||||
- ``POST /api/train/resume`` 解除挂起
|
||||
- ``POST /api/train/stop`` 请求优雅停止，超时后强杀进程树
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from collections import deque
|
||||
from dataclasses import dataclass
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from ..process_control import (DEFAULT_GRACE_S, spawn_kwargs,
|
||||
terminate_process_tree, write_control,
|
||||
write_json_atomic)
|
||||
from .analysis import (MAX_LOG_LINES, MAX_METRICS, describe_process, finite_float,
|
||||
process_pid, summarize_convergence)
|
||||
from .demo import DEMO_SCENARIOS, demo_snapshot
|
||||
from .schema import config_schema
|
||||
|
||||
__all__ = ["TrainingJob", "TrainingUI", "run_training_ui", "build_command",
|
||||
"summarize_convergence", "describe_process", "discover_recovery"]
|
||||
|
||||
STATIC_DIR = Path(__file__).resolve().parent / "static"
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
TRAIN_SCRIPT = REPO_ROOT / "scripts" / "train_hrl_high.py"
|
||||
# 控制文件集中放在 output/ 下（已 gitignore），job 之间互不干扰
|
||||
CONTROL_DIR = REPO_ROOT / "output" / "hrl" / "ui" / "control"
|
||||
# 从发出停止请求到强杀之间留给训练脚本收尾的秒数
|
||||
STOP_GRACE_S = DEFAULT_GRACE_S
|
||||
STATIC_FILES = {
|
||||
"/": ("index.html", "text/html; charset=utf-8"),
|
||||
"/index.html": ("index.html", "text/html; charset=utf-8"),
|
||||
"/app.js": ("app.js", "application/javascript; charset=utf-8"),
|
||||
"/charts.js": ("charts.js", "application/javascript; charset=utf-8"),
|
||||
"/style.css": ("style.css", "text/css; charset=utf-8"),
|
||||
}
|
||||
|
||||
_SB3_SECTIONS = ("rollout/", "time/", "train/", "system/")
|
||||
# 路径/字符串参数只拒绝控制字符（含 \r\n\t 与 DEL），不做 shell 元字符过滤：
|
||||
# 命令走参数列表、不经过 shell，这里只是防日志与命令行被注入控制序列。
|
||||
# 注意必须用单反斜杠转义——写成 r"...\\x00" 会变成"字面反斜杠到反斜杠"的
|
||||
# 字符类，从而把所有含 `\` 或看起来像路径分隔的输入误判为非法。
|
||||
_SAFE_ARG_RE = re.compile(r"^[^\x00-\x1f\x7f]{1,512}$")
|
||||
|
||||
|
||||
def _normalize_metric(key: str) -> str:
|
||||
"""把 SB3 的 ``rollout/ep_rew_mean`` 与进度 JSON 的 ``ep_rew_mean``
|
||||
归一化到同一短名，便于前端绘制。"""
|
||||
key = key.strip().replace(" ", "_")
|
||||
if "/" in key:
|
||||
key = key.rsplit("/", 1)[-1]
|
||||
return key
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TrainConfig:
|
||||
problem: int = 3
|
||||
stage: int = 1
|
||||
timesteps: int = 2048
|
||||
seed: int = 320001
|
||||
envs: int = 1
|
||||
vec_env: str = "dummy"
|
||||
device: str = "auto"
|
||||
config_profile: str = "production"
|
||||
reward_profile: str = "time"
|
||||
model: str = "output/hrl/ui/q3.zip"
|
||||
init: str | None = None
|
||||
resume: bool = False
|
||||
force: bool = False
|
||||
eval_episodes: int = 2
|
||||
rollout_steps: int = 128
|
||||
batch_size: int = 128
|
||||
n_epochs: int = 3
|
||||
max_wall_s: float = 1200.0
|
||||
local_selector: str = "geometry"
|
||||
local_model: str | None = None
|
||||
# 落盘/恢复：默认值与 scripts/train_hrl_high.py 保持一致
|
||||
checkpoint_every: int = 20000
|
||||
checkpoint_keep: int = 3
|
||||
recovery_every: int = 2000
|
||||
verbose: int = 1
|
||||
|
||||
@classmethod
|
||||
def from_payload(cls, payload: dict[str, Any]) -> "TrainConfig":
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("请求体必须是 JSON 对象")
|
||||
defaults = cls()
|
||||
data: dict[str, Any] = {}
|
||||
# 用 dataclasses.fields 而不是 cls.__dataclass_fields__：后者是内部
|
||||
# 实现细节，pylint 看不到它，也无法保证未来版本仍存在。
|
||||
for field in dataclasses.fields(cls):
|
||||
if field.name in payload and payload[field.name] is not None:
|
||||
data[field.name] = payload[field.name]
|
||||
cfg = cls(**{**defaults.__dict__, **data})
|
||||
cfg.validate()
|
||||
return cfg
|
||||
|
||||
def validate(self) -> None:
|
||||
if self.problem not in (3, 4):
|
||||
raise ValueError("problem 必须是 3 或 4")
|
||||
if self.stage not in (1, 2, 3, 4):
|
||||
raise ValueError("stage 必须在 1..4")
|
||||
if not 1 <= int(self.timesteps) <= 10_000_000:
|
||||
raise ValueError("timesteps 必须在 1..10,000,000")
|
||||
if bool(self.resume) and bool(self.force):
|
||||
raise ValueError("--resume 与 --force 互斥：续训不可能同时覆盖")
|
||||
if not 1 <= int(self.envs) <= 32:
|
||||
raise ValueError("envs 必须在 1..32")
|
||||
if self.vec_env not in ("dummy", "subproc"):
|
||||
raise ValueError("vec_env 必须是 dummy 或 subproc")
|
||||
if self.device not in ("auto", "cpu", "cuda", "mps"):
|
||||
raise ValueError("device 必须是 auto/cpu/cuda/mps")
|
||||
if self.config_profile not in ("production", "coarse-pretrain"):
|
||||
raise ValueError("config_profile 非法")
|
||||
if self.reward_profile not in ("time", "prompt"):
|
||||
raise ValueError("reward_profile 非法")
|
||||
if self.local_selector not in ("geometry", "rl"):
|
||||
raise ValueError("local_selector 非法")
|
||||
if not 1 <= int(self.rollout_steps) <= int(self.timesteps):
|
||||
raise ValueError("rollout_steps 必须在 1..timesteps")
|
||||
if not 1 <= int(self.n_epochs) <= 100:
|
||||
raise ValueError("n_epochs 必须在 1..100")
|
||||
if not 0.0 < float(self.max_wall_s) <= 7200.0:
|
||||
raise ValueError("max_wall_s 必须在 0..7200 秒")
|
||||
if int(self.eval_episodes) < 0:
|
||||
raise ValueError("eval_episodes 不能为负")
|
||||
if not 0 <= int(self.checkpoint_every) <= 10_000_000:
|
||||
raise ValueError("checkpoint_every 必须在 0..10,000,000")
|
||||
if not 1 <= int(self.checkpoint_keep) <= 100:
|
||||
raise ValueError("checkpoint_keep 必须在 1..100")
|
||||
if not 0 <= int(self.recovery_every) <= 10_000_000:
|
||||
raise ValueError("recovery_every 必须在 0..10,000,000")
|
||||
for label, value in (("model", self.model), ("init", self.init),
|
||||
("local_model", self.local_model)):
|
||||
if value is None:
|
||||
continue
|
||||
if not isinstance(value, str) or not _SAFE_ARG_RE.match(value):
|
||||
raise ValueError(f"{label} 路径含非法字符")
|
||||
|
||||
@property
|
||||
def model_path(self) -> Path:
|
||||
return (REPO_ROOT / self.model).resolve() if not Path(self.model).is_absolute() \
|
||||
else Path(self.model).resolve()
|
||||
|
||||
def command(self) -> list[str]:
|
||||
return build_command(self)
|
||||
|
||||
|
||||
def _int_query(params: dict[str, list[str]], name: str, default: int | None,
|
||||
low: int, high: int, *, optional: bool = False) -> int | None:
|
||||
"""解析整数查询参数；越界或非整数一律报错，不用静默钳制掩盖调用错误。"""
|
||||
if name not in params:
|
||||
return None if optional else default
|
||||
raw = (params[name] or [""])[0].strip()
|
||||
if raw == "":
|
||||
return None if optional else default
|
||||
try:
|
||||
value = int(raw)
|
||||
except ValueError:
|
||||
raise ValueError(f"{name} 必须是整数") from None
|
||||
if not low <= value <= high:
|
||||
raise ValueError(f"{name} 必须在 {low}..{high}")
|
||||
return value
|
||||
|
||||
|
||||
def build_command(config: TrainConfig) -> list[str]:
|
||||
if not TRAIN_SCRIPT.is_file():
|
||||
raise FileNotFoundError(f"训练脚本不存在: {TRAIN_SCRIPT}")
|
||||
cmd = [
|
||||
sys.executable, str(TRAIN_SCRIPT),
|
||||
"--problem", str(config.problem),
|
||||
"--stage", str(config.stage),
|
||||
"--timesteps", str(config.timesteps),
|
||||
"--seed", str(config.seed),
|
||||
"--envs", str(config.envs),
|
||||
"--vec-env", config.vec_env,
|
||||
"--device", config.device,
|
||||
"--config-profile", config.config_profile,
|
||||
"--reward-profile", config.reward_profile,
|
||||
"--model", str(config.model_path),
|
||||
"--eval-episodes", str(config.eval_episodes),
|
||||
"--rollout-steps", str(config.rollout_steps),
|
||||
"--batch-size", str(config.batch_size),
|
||||
"--n-epochs", str(config.n_epochs),
|
||||
"--max-wall-s", str(config.max_wall_s),
|
||||
"--local-selector", config.local_selector,
|
||||
"--checkpoint-every", str(config.checkpoint_every),
|
||||
"--checkpoint-keep", str(config.checkpoint_keep),
|
||||
"--recovery-every", str(config.recovery_every),
|
||||
"--progress-json",
|
||||
"--verbose", str(config.verbose),
|
||||
]
|
||||
if config.resume:
|
||||
cmd.append("--resume")
|
||||
if config.force:
|
||||
cmd.append("--force")
|
||||
if config.init:
|
||||
cmd.extend(["--init", str((REPO_ROOT / config.init).resolve()
|
||||
if not Path(config.init).is_absolute()
|
||||
else config.init)])
|
||||
if config.local_model:
|
||||
cmd.extend(["--local-model", str(config.local_model)])
|
||||
return cmd
|
||||
|
||||
|
||||
def discover_recovery(run_dir: Path, model_path: Path) -> dict[str, Any]:
|
||||
"""找出"现在就能继续训练"的那份产物。
|
||||
|
||||
优先级：
|
||||
1. ``<run_dir>/resume.json``——训练脚本写的权威指针（校验目标文件真实存在）；
|
||||
2. 否则按 mtime 取最新：最终模型 / 滚动恢复点 / 历史 checkpoint / 异常兜底。
|
||||
|
||||
只认**已完整落盘**的文件：模型的 zip 与 json 都是原子替换写的，所以不会
|
||||
出现"文件在、但打不开"的半截快照。
|
||||
"""
|
||||
run_dir = Path(run_dir)
|
||||
model_path = Path(model_path)
|
||||
hint_file = run_dir / "resume.json"
|
||||
hint = _read_json_file(hint_file)
|
||||
|
||||
candidates: list[tuple[float, str, str]] = []
|
||||
|
||||
def _consider(path: Path, reason: str) -> None:
|
||||
try:
|
||||
stat = path.stat()
|
||||
except OSError:
|
||||
return
|
||||
if stat.st_size > 0:
|
||||
candidates.append((stat.st_mtime, str(path), reason))
|
||||
|
||||
_consider(model_path, "final_model")
|
||||
ckpt_dir = run_dir / "checkpoints"
|
||||
if ckpt_dir.is_dir():
|
||||
_consider(ckpt_dir / f"{model_path.stem}_ckpt_latest.zip", "recovery")
|
||||
for path in ckpt_dir.glob(f"{model_path.stem}_ckpt_[0-9]*.zip"):
|
||||
if path.stem.rsplit("_", 1)[-1].isdigit():
|
||||
_consider(path, "checkpoint")
|
||||
_consider(run_dir / f"{model_path.stem}_emergency.zip", "emergency")
|
||||
|
||||
state: dict[str, Any] = {
|
||||
"can_resume": False, "model": None, "timesteps": None,
|
||||
"reason": "none", "updated_unix_s": None, "source": "none",
|
||||
"hint_file": str(hint_file), "candidates": len(candidates),
|
||||
}
|
||||
if hint and hint.get("model") and Path(str(hint["model"])).is_file():
|
||||
state.update({
|
||||
"model": str(hint["model"]),
|
||||
"timesteps": _int_or_none(hint.get("timesteps")),
|
||||
"reason": str(hint.get("reason") or "resume_hint"),
|
||||
"updated_unix_s": _float_or_none(hint.get("created_unix_s")),
|
||||
"source": "resume.json",
|
||||
})
|
||||
elif candidates:
|
||||
mtime, path, reason = max(candidates, key=lambda item: item[0])
|
||||
state.update({
|
||||
"model": path, "reason": reason,
|
||||
"updated_unix_s": mtime, "source": "scan",
|
||||
})
|
||||
if state["model"]:
|
||||
state["can_resume"] = Path(str(state["model"])).is_file()
|
||||
return state
|
||||
|
||||
|
||||
def _read_json_file(path: Path) -> dict[str, Any] | None:
|
||||
"""读 JSON 文件；缺失或损坏返回 None（恢复指针不是关键路径，不能因此崩）。"""
|
||||
try:
|
||||
payload = json.loads(Path(path).read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError, UnicodeDecodeError):
|
||||
return None
|
||||
return payload if isinstance(payload, dict) else None
|
||||
|
||||
|
||||
def _int_or_none(value: Any) -> int | None:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _float_or_none(value: Any) -> float | None:
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
class TrainingJob:
|
||||
"""单个训练子进程的生命周期、日志和指标缓存。
|
||||
|
||||
``control_dir`` 是控制文件的落地目录；默认与模型同目录，服务端则统一放到
|
||||
``output/hrl/ui/control``（已被 gitignore）。
|
||||
"""
|
||||
|
||||
def __init__(self, config: TrainConfig, control_dir: Path | None = None):
|
||||
self.config = config
|
||||
self.job_id = f"job-{int(time.time() * 1000)}"
|
||||
self.control_path = (Path(control_dir) if control_dir is not None
|
||||
else config.model_path.parent) / \
|
||||
f"{self.job_id}.control.json"
|
||||
# 控制文件是暂停/停止的唯一可靠通道：Windows 上没有 SIGSTOP，
|
||||
# 且 os.kill(SIGTERM) 会退化成硬杀，信号无法承载"优雅收尾"语义。
|
||||
self.command = config.command() + ["--control-file", str(self.control_path)]
|
||||
self.created_at = time.time()
|
||||
self.started_at: float | None = None
|
||||
self.finished_at: float | None = None
|
||||
self.returncode: int | None = None
|
||||
# created/running/paused/stopping/stopped/exited/failed
|
||||
self.status = "created"
|
||||
self.error: str | None = None
|
||||
self.forced_kill = False # 停止请求超时后是否动用了强杀
|
||||
self.process: subprocess.Popen | None = None
|
||||
self.logs: deque[str] = deque(maxlen=MAX_LOG_LINES)
|
||||
self.metrics: deque[dict[str, Any]] = deque(maxlen=MAX_METRICS)
|
||||
self.monitor: list[dict[str, Any]] = []
|
||||
self._lock = threading.RLock()
|
||||
self._reader: threading.Thread | None = None
|
||||
self._stopper: threading.Thread | None = None
|
||||
self._table_sections: list[tuple[str, dict[str, Any]]] = []
|
||||
self._table_steps: float | None = None # 当前表的 total_timesteps
|
||||
self._pending_table: dict[str, Any] | None = None # 未见刻度的半张表
|
||||
self._last_table_steps: float | None = None # 上一张表已知刻度（截断兜底）
|
||||
self._last_json_steps: float | None = None # 最近一条 progress JSON 的步数
|
||||
self._last_monitor_sig: tuple[int, int] | None = None
|
||||
|
||||
# ------------------------------------------------------------ lifecycle
|
||||
def start(self) -> None:
|
||||
with self._lock:
|
||||
if self.process is not None and self.process.poll() is None:
|
||||
raise RuntimeError("已有训练任务在运行")
|
||||
# 先落一份"不暂停、不停止"的初始控制文件，训练脚本首步就能读到；
|
||||
# 否则脚本要等到第一次轮询才区分"文件缺失"与"无指令"。
|
||||
write_control(self.control_path, paused=False, stop=False)
|
||||
env = os.environ.copy()
|
||||
env["PYTHONUNBUFFERED"] = "1"
|
||||
env["HRL_UI_JOB_ID"] = self.job_id
|
||||
self.process = subprocess.Popen( # pylint: disable=consider-using-with
|
||||
self.command,
|
||||
cwd=str(REPO_ROOT),
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
bufsize=1,
|
||||
env=env,
|
||||
**spawn_kwargs(),
|
||||
)
|
||||
self.started_at = time.time()
|
||||
self.status = "running"
|
||||
self._reader = threading.Thread(
|
||||
target=self._read_loop, name=f"{self.job_id}-reader", daemon=True)
|
||||
self._reader.start()
|
||||
|
||||
def pause(self) -> bool:
|
||||
"""请求训练进程在下一个 step 边界挂起（跨平台：控制文件）。"""
|
||||
with self._lock:
|
||||
process = self.process
|
||||
if process is None or process.poll() is not None:
|
||||
return False
|
||||
if self.status not in ("running", "paused"):
|
||||
return False
|
||||
write_control(self.control_path, paused=True, stop=False)
|
||||
self.status = "paused"
|
||||
return True
|
||||
|
||||
def resume(self) -> bool:
|
||||
"""解除挂起；训练从暂停的那个 step 继续。"""
|
||||
with self._lock:
|
||||
process = self.process
|
||||
if process is None or process.poll() is not None:
|
||||
return False
|
||||
if self.status not in ("paused", "running"):
|
||||
return False
|
||||
write_control(self.control_path, paused=False, stop=False)
|
||||
self.status = "running"
|
||||
return True
|
||||
|
||||
def stop(self) -> bool:
|
||||
"""请求优雅停止；超时后由看门狗强杀整棵进程树。
|
||||
|
||||
控制文件先于信号：Windows 上 SIGTERM 等价于 TerminateProcess，训练
|
||||
脚本"保存模型 + 写元数据"的收尾分支不会执行；控制文件则两个平台都能
|
||||
命中同一段协作式收尾。强杀放到独立线程，避免 HTTP 请求被 grace 等待卡住。
|
||||
"""
|
||||
with self._lock:
|
||||
process = self.process
|
||||
if process is None or process.poll() is not None:
|
||||
return False
|
||||
write_control(self.control_path, paused=False, stop=True)
|
||||
self.status = "stopping"
|
||||
if self._stopper is None or not self._stopper.is_alive():
|
||||
self._stopper = threading.Thread(
|
||||
target=self._stop_watchdog, args=(process,),
|
||||
name=f"{self.job_id}-stopper", daemon=True)
|
||||
self._stopper.start()
|
||||
return True
|
||||
|
||||
def _stop_watchdog(self, process: subprocess.Popen) -> None:
|
||||
"""等 grace 让协作式停止生效，超时强杀并记下恢复点。"""
|
||||
forced = terminate_process_tree(process, grace_s=STOP_GRACE_S)
|
||||
with self._lock:
|
||||
self.forced_kill = forced
|
||||
if forced and not self.error:
|
||||
self.error = (f"训练进程未在 {STOP_GRACE_S:.0f}s 内响应停止请求，"
|
||||
f"已强制结束进程树")
|
||||
if forced:
|
||||
self._write_forced_stop(process.pid)
|
||||
|
||||
def _write_forced_stop(self, pid: int) -> None:
|
||||
"""强杀后落一份 `<模型目录>/forced_stop.json`，写明"从哪继续"。
|
||||
|
||||
强杀没有给训练脚本任何收尾机会，所以这里由前端补上审计与恢复指引：
|
||||
磁盘上仍有原子落盘的滚动恢复点 / 历史 checkpoint，``recovery.model``
|
||||
就是可直接 ``--resume`` 的目标。
|
||||
"""
|
||||
run_dir = self.config.model_path.parent
|
||||
try:
|
||||
recovery = discover_recovery(run_dir, self.config.model_path)
|
||||
write_json_atomic(run_dir / "forced_stop.json", {
|
||||
"event": "forced_stop",
|
||||
"job_id": self.job_id,
|
||||
"pid": int(pid),
|
||||
"created_unix_s": time.time(),
|
||||
"reason": "grace_timeout",
|
||||
"grace_s": STOP_GRACE_S,
|
||||
"recovery": recovery,
|
||||
"note": ("训练进程未响应控制文件，已强杀整棵进程树；"
|
||||
"把 recovery.model 作为 --resume 目标即可继续"
|
||||
"（timesteps 为追加步数）"),
|
||||
})
|
||||
except OSError as exc: # 标记文件写不了也不能影响停止流程
|
||||
with self._lock:
|
||||
if not self.error:
|
||||
self.error = f"强杀后写恢复标记失败：{exc}"
|
||||
|
||||
def wait_for_exit(self, timeout: float | None = None) -> bool:
|
||||
"""等待停止流程收尾（优雅退出或强杀）；供前端进程退出前调用。
|
||||
|
||||
训练子进程在独立进程组里，前端先退出并不会带走它。若不在这里等，
|
||||
前端一退出，子进程会成为孤儿——虽然控制文件仍会让它自行收尾，但
|
||||
日志与退出码就没人接了。
|
||||
"""
|
||||
thread = self._stopper
|
||||
if thread is not None:
|
||||
thread.join(timeout)
|
||||
process = self.process
|
||||
return process is None or process.poll() is not None
|
||||
|
||||
def _read_loop(self) -> None:
|
||||
process = self.process
|
||||
assert process is not None and process.stdout is not None
|
||||
try:
|
||||
for raw in process.stdout:
|
||||
line = raw.rstrip("\n")
|
||||
self._on_line(line)
|
||||
process.wait()
|
||||
except Exception as exc: # pragma: no cover - 子进程读取异常
|
||||
with self._lock:
|
||||
self.error = f"{type(exc).__name__}: {exc}"
|
||||
finally:
|
||||
with self._lock:
|
||||
self.finished_at = time.time()
|
||||
if self.returncode is None:
|
||||
self.returncode = process.poll()
|
||||
if self.status == "stopping":
|
||||
self.status = "stopped"
|
||||
elif self.returncode == 0:
|
||||
self.status = "exited"
|
||||
else:
|
||||
self.status = "failed"
|
||||
self._flush_table()
|
||||
# 最后半张表（Ctrl+C / 崩溃时没打印 total_timesteps）按末尾
|
||||
# 已知刻度兜底提交，宁可标注来源也不丢最后一段曲线
|
||||
self._flush_pending_with_fallback()
|
||||
|
||||
def _on_line(self, line: str) -> None:
|
||||
with self._lock:
|
||||
self.logs.append(line)
|
||||
self._sync_control_events(line)
|
||||
self._parse_progress_json(line)
|
||||
self._parse_sb3_table(line)
|
||||
self._refresh_monitor()
|
||||
|
||||
def _sync_control_events(self, line: str) -> None:
|
||||
"""用训练脚本的回显校正前端状态。
|
||||
|
||||
正常情况下状态由前端自己设置；但外部直接改控制文件、或脚本比前端更早
|
||||
感知到指令时，以脚本回显的 ``paused`` / ``resumed`` / ``stop_requested``
|
||||
为准更真实。
|
||||
"""
|
||||
stripped = line.strip()
|
||||
if not stripped.startswith("{") or '"event"' not in stripped:
|
||||
return
|
||||
try:
|
||||
payload = json.loads(stripped)
|
||||
except ValueError:
|
||||
return
|
||||
if not isinstance(payload, dict):
|
||||
return
|
||||
event = payload.get("event")
|
||||
if event not in ("paused", "resumed", "stop_requested"):
|
||||
return
|
||||
with self._lock:
|
||||
if event == "paused" and self.status == "running":
|
||||
self.status = "paused"
|
||||
elif event == "resumed" and self.status == "paused":
|
||||
self.status = "running"
|
||||
elif event == "stop_requested" and self.status in ("running", "paused"):
|
||||
self.status = "stopping"
|
||||
|
||||
# ------------------------------------------------------------ parsers
|
||||
def _parse_progress_json(self, line: str) -> None:
|
||||
stripped = line.strip()
|
||||
if not stripped.startswith("{") or "progress" not in stripped:
|
||||
return
|
||||
try:
|
||||
payload = json.loads(stripped)
|
||||
except ValueError:
|
||||
return
|
||||
if not isinstance(payload, dict) or payload.get("event") != "progress":
|
||||
return
|
||||
point = {_normalize_metric(str(key)): value
|
||||
for key, value in payload.items() if key != "event"}
|
||||
steps = finite_float(point.get("timesteps"))
|
||||
if steps is None:
|
||||
steps = finite_float(point.get("total_timesteps"))
|
||||
if steps is not None:
|
||||
with self._lock:
|
||||
self._last_json_steps = steps
|
||||
self._append_metric(point)
|
||||
|
||||
def _parse_sb3_table(self, line: str) -> None:
|
||||
"""兼容 SB3 verbose=1 的 ``| key | value |`` 表格。
|
||||
|
||||
SB3 每张表按 rollout/ → time/ → train/ → system/ 分段，``total_timesteps``
|
||||
出现在 time/ 段——也就是要等同一张表的 rollout/ 行先读进来之后才拿到
|
||||
横轴。因此不能在段头立刻提交：那会把还没有刻度的 rollout/ 行丢掉，或
|
||||
把它们算到上一张表的刻度上。
|
||||
|
||||
规则：
|
||||
- 每次"回到 rollout/ 段"视为新一张表开始，先把上一张已封闭的表提交；
|
||||
- 表格分隔线表示本表结束，此时才允许用上一张表的刻度兜底（截断表）；
|
||||
- 从未封闭过的半张表一直挂到进程结束才兜底提交，避免用旧刻度冒充。
|
||||
"""
|
||||
if not line.startswith("|"):
|
||||
if line.startswith("-"):
|
||||
self._flush_table()
|
||||
return
|
||||
cells = [cell.strip() for cell in line.split("|")]
|
||||
if len(cells) < 4:
|
||||
return
|
||||
raw_key, raw_value = cells[1], cells[2]
|
||||
if raw_key in _SB3_SECTIONS:
|
||||
if cells[0] != "":
|
||||
return
|
||||
with self._lock:
|
||||
if raw_key == _SB3_SECTIONS[0] and self._table_sections:
|
||||
self._flush_table()
|
||||
if not self._table_sections or self._table_sections[-1][0] != raw_key:
|
||||
self._table_sections.append((raw_key, {}))
|
||||
return
|
||||
key = _normalize_metric(raw_key)
|
||||
value = finite_float(raw_value)
|
||||
if not key or value is None:
|
||||
return
|
||||
with self._lock:
|
||||
if not self._table_sections:
|
||||
self._table_sections.append((_SB3_SECTIONS[0], {}))
|
||||
self._table_sections[-1][1][key] = value
|
||||
if key == "total_timesteps":
|
||||
self._table_steps = value
|
||||
|
||||
def _flush_table(self) -> None:
|
||||
"""提交缓存中的 SB3 表。
|
||||
|
||||
刻度优先级：本表的 ``total_timesteps`` > 上一张表的已知刻度（仅在遇到
|
||||
分隔线即"确认整表结束"时才允许兜底，见 ``_on_line``）。完全没有刻度
|
||||
就丢弃——伪造横轴会在曲线上画出重复的幽灵点。与 JSON 进度行同一步数时
|
||||
跳过表格版本（JSON 字段更全），表格只用于补 clip_fraction/learning_rate。
|
||||
"""
|
||||
with self._lock:
|
||||
sections = self._table_sections
|
||||
steps = self._table_steps
|
||||
self._table_sections = []
|
||||
self._table_steps = None
|
||||
last_json = self._last_json_steps
|
||||
if not sections:
|
||||
return
|
||||
table: dict[str, Any] = {}
|
||||
for _name, items in sections:
|
||||
table.update(items)
|
||||
if steps is None:
|
||||
steps = finite_float(table.get("total_timesteps"))
|
||||
if steps is None:
|
||||
# 半张表：交给调用方决定是否已确认结束（终止时按末尾兜底提交）
|
||||
with self._lock:
|
||||
self._pending_table = table
|
||||
return
|
||||
with self._lock:
|
||||
self._last_table_steps = steps
|
||||
if last_json is not None:
|
||||
tolerance = max(1.0, 0.25 * steps / max(1.0, self.config.rollout_steps))
|
||||
if abs(steps - last_json) <= tolerance:
|
||||
return
|
||||
if "timesteps" not in table:
|
||||
table["timesteps"] = steps
|
||||
self._append_metric(table)
|
||||
|
||||
def _flush_pending_with_fallback(self) -> None:
|
||||
"""进程结束时提交最后半张表：用上一次已知刻度兜底，避免丢数据。"""
|
||||
with self._lock:
|
||||
pending = self._pending_table
|
||||
self._pending_table = None
|
||||
steps = self._last_table_steps
|
||||
if not pending or steps is None:
|
||||
return
|
||||
if "timesteps" not in pending:
|
||||
pending = {**pending, "timesteps": steps}
|
||||
self._append_metric(pending)
|
||||
|
||||
def _append_metric(self, point: dict[str, Any]) -> None:
|
||||
clean: dict[str, Any] = {}
|
||||
for key, value in point.items():
|
||||
number = finite_float(value)
|
||||
clean[key] = number if number is not None else value
|
||||
# 补齐 timesteps：progress JSON 优先，SB3 表用 total_timesteps。
|
||||
if "timesteps" not in clean and "total_timesteps" in clean:
|
||||
clean["timesteps"] = clean["total_timesteps"]
|
||||
with self._lock:
|
||||
if self.metrics and clean == self.metrics[-1]:
|
||||
return
|
||||
self.metrics.append(clean)
|
||||
|
||||
def _refresh_monitor(self) -> None:
|
||||
"""训练脚本写入的 Monitor CSV：作为逐 episode 奖励的备用来源。"""
|
||||
parent = self.config.model_path.parent
|
||||
files = sorted(parent.glob("high_curve_*_env*.monitor.csv")) \
|
||||
if parent.is_dir() else []
|
||||
if not files:
|
||||
return
|
||||
signature = (len(files), max(path.stat().st_mtime_ns for path in files))
|
||||
if signature == self._last_monitor_sig:
|
||||
return
|
||||
rows: list[dict[str, Any]] = []
|
||||
for path in files:
|
||||
try:
|
||||
lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
|
||||
except OSError:
|
||||
continue
|
||||
header: list[str] | None = None
|
||||
for line in lines:
|
||||
if line.startswith("#") or not line.strip():
|
||||
continue
|
||||
if header is None:
|
||||
header = [item.strip() for item in line.split(",")]
|
||||
continue
|
||||
values = line.split(",")
|
||||
if len(values) < len(header):
|
||||
continue
|
||||
row = {}
|
||||
valid = True
|
||||
for key, raw in zip(header, values):
|
||||
number = finite_float(raw)
|
||||
if number is None:
|
||||
valid = False
|
||||
break
|
||||
row[key] = number
|
||||
if valid:
|
||||
rows.append(row)
|
||||
with self._lock:
|
||||
self.monitor = rows[-MAX_METRICS:]
|
||||
self._last_monitor_sig = signature
|
||||
|
||||
# ------------------------------------------------------------ snapshot
|
||||
def snapshot(self, *, log_tail: int = 400, metric_limit: int = 2000,
|
||||
after_metric: int | None = None) -> dict[str, Any]:
|
||||
with self._lock:
|
||||
now = self.finished_at or time.time()
|
||||
elapsed = (now - self.started_at) if self.started_at else 0.0
|
||||
logs = list(self.logs)[-max(1, int(log_tail)):]
|
||||
total = len(self.metrics)
|
||||
if after_metric is None:
|
||||
metrics = list(self.metrics)[-max(1, int(metric_limit)):]
|
||||
offset = total - len(metrics)
|
||||
else:
|
||||
start = max(0, min(int(after_metric), total))
|
||||
metrics = list(self.metrics)[start:]
|
||||
offset = start
|
||||
# metric_end 是本批点在序列中的绝对末端下标：服务端 deque 满 5000
|
||||
# 后会丢弃最老的点，客户端只有拿到这个值才能区分"续接"与"服务端
|
||||
# 已滑动窗口"，否则同一批点会被重复追加。
|
||||
metric_end = offset + len(metrics)
|
||||
monitor = self.monitor[-max(1, int(metric_limit)):]
|
||||
config = dict(self.config.__dict__)
|
||||
command = list(self.command)
|
||||
model_path = str(self.config.model_path)
|
||||
payload = {
|
||||
"job_id": self.job_id,
|
||||
"status": self.status,
|
||||
"pid": process_pid(self.process),
|
||||
"created_at": self.created_at,
|
||||
"started_at": self.started_at,
|
||||
"finished_at": self.finished_at,
|
||||
"elapsed_s": round(elapsed, 3),
|
||||
"returncode": self.returncode,
|
||||
"error": self.error,
|
||||
"forced_kill": self.forced_kill,
|
||||
"command": command,
|
||||
"model": model_path,
|
||||
"config": config,
|
||||
"log_lines": logs,
|
||||
"log_total": len(self.logs),
|
||||
"metrics": metrics,
|
||||
"metric_offset": offset,
|
||||
"metric_end": metric_end,
|
||||
"metric_total": total,
|
||||
"monitor": monitor,
|
||||
}
|
||||
payload["convergence"] = summarize_convergence(metrics, monitor)
|
||||
payload["process"] = describe_process(payload)
|
||||
# 恢复指针在锁外做文件扫描：即使训练被强杀，这里也能给出可续训的产物
|
||||
payload["recovery"] = discover_recovery(
|
||||
self.config.model_path.parent, self.config.model_path)
|
||||
return payload
|
||||
|
||||
|
||||
class TrainingUI:
|
||||
"""训练前端管理器：持有唯一 TrainingJob 与 HTTP 服务器。"""
|
||||
|
||||
def __init__(self, ui_port: int = 8036):
|
||||
self.ui_port = int(ui_port)
|
||||
self.job: TrainingJob | None = None
|
||||
self._lock = threading.RLock()
|
||||
try:
|
||||
self.httpd = ThreadingHTTPServer(("127.0.0.1", self.ui_port), _UIHandler)
|
||||
except OSError as exc:
|
||||
raise OSError(
|
||||
f"无法在 127.0.0.1:{self.ui_port} 启动训练前端（{exc}）；"
|
||||
f"换一个 --ui-port 或先结束占用该端口的进程") from exc
|
||||
self.httpd.daemon_threads = True
|
||||
self.httpd.training_ui = self
|
||||
self._serving = False
|
||||
|
||||
# ------------------------------------------------------------ API
|
||||
def start(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
config = TrainConfig.from_payload(payload)
|
||||
with self._lock:
|
||||
if self.job is not None and self.job.status in (
|
||||
"created", "running", "paused", "stopping"):
|
||||
raise RuntimeError("已有训练任务在运行，请先停止")
|
||||
self.job = TrainingJob(config, control_dir=CONTROL_DIR)
|
||||
try:
|
||||
self.job.start()
|
||||
except Exception:
|
||||
self.job.status = "failed"
|
||||
self.job.error = "训练进程启动失败"
|
||||
raise
|
||||
# 与 snapshot() 同形：HTTP 层直接把返回值当响应体，不再二次拆包
|
||||
return {"job": self.job.snapshot(log_tail=1)}
|
||||
|
||||
def pause(self) -> bool:
|
||||
with self._lock:
|
||||
return bool(self.job and self.job.pause())
|
||||
|
||||
def resume(self) -> bool:
|
||||
with self._lock:
|
||||
return bool(self.job and self.job.resume())
|
||||
|
||||
def stop(self) -> bool:
|
||||
with self._lock:
|
||||
return bool(self.job and self.job.stop())
|
||||
|
||||
def snapshot(self, *, log_tail: int, metric_limit: int,
|
||||
after_metric: int | None = None) -> dict[str, Any]:
|
||||
with self._lock:
|
||||
if self.job is None:
|
||||
return {"job": None, "message": "尚未启动训练"}
|
||||
return {"job": self.job.snapshot(log_tail=log_tail,
|
||||
metric_limit=metric_limit,
|
||||
after_metric=after_metric)}
|
||||
|
||||
@property
|
||||
def is_serving(self) -> bool:
|
||||
return self._serving
|
||||
|
||||
def serve_forever(self) -> None:
|
||||
self._serving = True
|
||||
try:
|
||||
self.httpd.serve_forever()
|
||||
finally:
|
||||
self._serving = False
|
||||
|
||||
def shutdown(self) -> None:
|
||||
job = self.job
|
||||
if job is not None and job.stop():
|
||||
# 留出收尾落盘的时间，避免前端先退出把训练子进程留成孤儿
|
||||
job.wait_for_exit(timeout=STOP_GRACE_S + 5.0)
|
||||
if self._serving:
|
||||
self.httpd.shutdown()
|
||||
self.httpd.server_close()
|
||||
|
||||
|
||||
class _UIHandler(BaseHTTPRequestHandler):
|
||||
protocol_version = "HTTP/1.1"
|
||||
server_version = "MCM26B-Training-UI/1.0"
|
||||
|
||||
def log_message(self, format, *args): # noqa: A002 # pylint: disable=redefined-builtin
|
||||
"""覆写 stdlib 签名以吞掉访问日志；参数名沿用 ``format`` 不能改。"""
|
||||
|
||||
@property
|
||||
def ui(self) -> TrainingUI:
|
||||
return self.server.training_ui # type: ignore[attr-defined]
|
||||
|
||||
def do_GET(self):
|
||||
parts = urlsplit(self.path)
|
||||
if parts.path in STATIC_FILES:
|
||||
self._serve_static(parts.path)
|
||||
elif parts.path == "/api/train/schema":
|
||||
self._send_json(200, config_schema())
|
||||
elif parts.path == "/api/train/state":
|
||||
params = parse_qs(parts.query)
|
||||
if (params.get("preview") or ["0"])[0] in ("1", "true", "yes"):
|
||||
# 离线预览：返回合成快照，不触碰任何训练进程
|
||||
scenario = (params.get("scenario") or ["rising"])[0]
|
||||
if scenario not in DEMO_SCENARIOS:
|
||||
self._send_json(400, {"error": f"未知预览场景 {scenario}"})
|
||||
return
|
||||
self._send_json(200, {"job": demo_snapshot(log_tail=400,
|
||||
scenario=scenario)})
|
||||
return
|
||||
try:
|
||||
log_tail = _int_query(params, "log_tail", 400, 1, MAX_LOG_LINES)
|
||||
metric_limit = _int_query(params, "metric_limit", 2000, 1,
|
||||
MAX_METRICS)
|
||||
after = _int_query(params, "after_metric", None, 0,
|
||||
MAX_METRICS * 2, optional=True)
|
||||
except ValueError as exc:
|
||||
self._send_json(400, {"error": str(exc)})
|
||||
return
|
||||
self._send_json(200, self.ui.snapshot(
|
||||
log_tail=log_tail, metric_limit=metric_limit,
|
||||
after_metric=after))
|
||||
else:
|
||||
self._send_json(404, {"error": f"未知路径 {parts.path}"})
|
||||
|
||||
def do_POST(self):
|
||||
parts = urlsplit(self.path)
|
||||
payload = self._read_json()
|
||||
if payload is None:
|
||||
self._send_json(400, {"error": "请求体必须是合法 JSON"})
|
||||
return
|
||||
if not isinstance(payload, dict):
|
||||
self._send_json(400, {"error": "请求体必须是 JSON 对象"})
|
||||
return
|
||||
try:
|
||||
if parts.path == "/api/train/start":
|
||||
self._send_json(200, self.ui.start(payload))
|
||||
elif parts.path == "/api/train/pause":
|
||||
self._send_json(200, {"paused": self.ui.pause()})
|
||||
elif parts.path == "/api/train/resume":
|
||||
self._send_json(200, {"resumed": self.ui.resume()})
|
||||
elif parts.path == "/api/train/stop":
|
||||
self._send_json(200, {"stopped": self.ui.stop()})
|
||||
else:
|
||||
self._send_json(404, {"error": f"未知路径 {parts.path}"})
|
||||
except (ValueError, RuntimeError, FileNotFoundError,
|
||||
PermissionError) as exc:
|
||||
self._send_json(409, {"error": str(exc)})
|
||||
except OSError as exc:
|
||||
self._send_json(409, {"error": f"操作系统错误：{exc}"})
|
||||
except Exception as exc: # pragma: no cover - 兜底，避免连接被静默中断
|
||||
self._send_json(500, {"error": f"服务端内部错误：{type(exc).__name__}: {exc}"})
|
||||
|
||||
def _serve_static(self, path: str) -> None:
|
||||
filename, content_type = STATIC_FILES[path]
|
||||
data = (STATIC_DIR / filename).read_bytes()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", content_type)
|
||||
self.send_header("Content-Length", str(len(data)))
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
|
||||
def _read_json(self) -> dict[str, Any] | None:
|
||||
try:
|
||||
length = int(self.headers.get("Content-Length") or 0)
|
||||
except ValueError:
|
||||
return None
|
||||
if length <= 0 or length > 1_000_000:
|
||||
return {}
|
||||
data = self.rfile.read(length)
|
||||
try:
|
||||
return json.loads(data.decode("utf-8"))
|
||||
except (ValueError, UnicodeDecodeError):
|
||||
return None
|
||||
|
||||
def _send_json(self, status: int, payload: dict[str, Any]) -> None:
|
||||
data = json.dumps(payload, ensure_ascii=False, allow_nan=False).encode("utf-8")
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||||
self.send_header("Content-Length", str(len(data)))
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
|
||||
|
||||
def run_training_ui(argv=None) -> int:
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="HRL 模型训练前端（本地回环）")
|
||||
parser.add_argument("--ui-port", type=int, default=8036)
|
||||
parser.add_argument("--no-browser", action="store_true")
|
||||
parser.add_argument("--print-schema", action="store_true",
|
||||
help="打印配置契约（字段/分组/预设/指标）后退出")
|
||||
parser.add_argument("--print-preview", action="store_true",
|
||||
help="打印离线示例快照 JSON 后退出")
|
||||
args = parser.parse_args(argv)
|
||||
if args.print_schema:
|
||||
print(json.dumps(config_schema(), ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
if args.print_preview:
|
||||
print(json.dumps({"job": demo_snapshot(log_tail=200)},
|
||||
ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
try:
|
||||
ui = TrainingUI(ui_port=args.ui_port)
|
||||
except OSError as exc:
|
||||
print(f"[训练前端] {exc}", file=sys.stderr)
|
||||
return 2
|
||||
url = f"http://127.0.0.1:{args.ui_port}/"
|
||||
# flush=True：界面常用 `> log 2>&1 &` 方式后台启动，不刷新会看不到启动横幅
|
||||
print(f"训练前端已启动：{url}", flush=True)
|
||||
print(f" 配置契约：{url}api/train/schema", flush=True)
|
||||
print(f" 状态快照：{url}api/train/state", flush=True)
|
||||
print(f" 离线预览：{url}api/train/state?preview=1", flush=True)
|
||||
print(" 界面只监听 127.0.0.1；Ctrl+C 退出并停止训练子进程。", flush=True)
|
||||
if not args.no_browser:
|
||||
try:
|
||||
import webbrowser
|
||||
webbrowser.open(url)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
ui.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
finally:
|
||||
ui.shutdown()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
raise SystemExit(run_training_ui())
|
||||
@@ -1,841 +0,0 @@
|
||||
/* 训练前端主逻辑：纯原生 JS，无外部依赖。
|
||||
职责分工：
|
||||
- 表单由 /api/train/schema 的字段元数据生成，不在前端复制校验规则；
|
||||
- 训练数值全部取自 /api/train/state 快照（服务端已算好进度、吞吐、
|
||||
收敛判读），前端只做格式化与渲染，避免两套算法对不上；
|
||||
- 指标增量拉取（after_metric 游标），日志按尾部行数拉取。 */
|
||||
"use strict";
|
||||
|
||||
const $ = (id) => document.getElementById(id);
|
||||
const REFRESH_MS = 1500;
|
||||
// 与服务端 MAX_METRICS 对齐：本地缓存上限，避免长时间训练把内存吃满
|
||||
const METRIC_LIMIT = 5000;
|
||||
|
||||
const state = {
|
||||
schema: null,
|
||||
preview: false, // 离线预览：不再启动进程，只渲染示例快照
|
||||
polling: false,
|
||||
saving: false, // 配置表单是否处于"训练中"锁定态
|
||||
metrics: [], // 本地累积的进度点（按 timesteps 递增）
|
||||
metricCache: [], // metrics 的同一个数组（显式持有，避免 slice/concat 时丢引用）
|
||||
metricNext: 0, // 本地已吃到的下一个下标（= 服务端下次应返回的 offset）
|
||||
metricTotal: 0, // 服务端累计进度点总数（增量拉取的游标）
|
||||
chartKey: "ep_rew_mean",
|
||||
jobKey: null, // job_id 变化时清空本地缓存
|
||||
lastLogSignature: "",
|
||||
lastStatus: null, // 最近一次快照的状态：决定暂停按钮是"暂停"还是"继续"
|
||||
lastJob: null, // 最近一次快照：恢复按钮需要读 job.recovery
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------- 工具
|
||||
function escapeHtml(value) {
|
||||
return String(value == null ? "" : value)
|
||||
.replace(/[&<>"']/g, (c) => ({ "&": "&", "<": "<", ">": ">",
|
||||
'"': """, "'": "'" }[c]));
|
||||
}
|
||||
function fmtNumber(value, digits) {
|
||||
if (value === null || value === undefined || value === "" || Number.isNaN(Number(value))) return "—";
|
||||
const num = Number(value);
|
||||
const abs = Math.abs(num);
|
||||
if (abs !== 0 && (abs < 1e-3 || abs >= 1e5)) return num.toExponential(2);
|
||||
const d = digits === undefined ? (abs >= 100 ? 1 : abs >= 1 ? 2 : 4) : digits;
|
||||
return num.toFixed(d);
|
||||
}
|
||||
function fmtInt(value) {
|
||||
if (value === null || value === undefined || Number.isNaN(Number(value))) return "—";
|
||||
return Math.round(Number(value)).toLocaleString("en-US");
|
||||
}
|
||||
function fmtDuration(seconds) {
|
||||
if (seconds === null || seconds === undefined || !isFinite(seconds) || seconds < 0) return "—";
|
||||
const s = Math.round(seconds);
|
||||
const h = Math.floor(s / 3600), m = Math.floor((s % 3600) / 60), sec = s % 60;
|
||||
if (h > 0) return `${h}h${String(m).padStart(2, "0")}m`;
|
||||
if (m > 0) return `${m}m${String(sec).padStart(2, "0")}s`;
|
||||
return `${sec}s`;
|
||||
}
|
||||
function fmtClock(epochSeconds) {
|
||||
if (!epochSeconds) return "—";
|
||||
const d = new Date(epochSeconds * 1000);
|
||||
return `${String(d.getHours()).padStart(2, "0")}:${String(d.getMinutes()).padStart(2, "0")}:${String(d.getSeconds()).padStart(2, "0")}`;
|
||||
}
|
||||
function chip(id, label, value, tone) {
|
||||
const el = $(id);
|
||||
if (!el) return;
|
||||
el.className = `chip ${tone || "gray"}`;
|
||||
el.innerHTML = `${label}<b>${escapeHtml(value)}</b>`;
|
||||
}
|
||||
function setText(id, text) { const el = $(id); if (el) el.textContent = text; }
|
||||
|
||||
// ---------------------------------------------------------------- 启动
|
||||
async function boot() {
|
||||
try {
|
||||
const res = await fetch("/api/train/schema");
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
state.schema = await res.json();
|
||||
} catch (err) {
|
||||
state.schema = { fields: [], groups: [], presets: [], metrics: [] };
|
||||
showBanner("bad", `无法读取配置契约（${err.message}）：请确认已通过 python -m mcm26b.training_ui 启动前端。`);
|
||||
}
|
||||
buildForm();
|
||||
await poll();
|
||||
bindChartInteractions();
|
||||
setInterval(poll, REFRESH_MS);
|
||||
}
|
||||
|
||||
function showBanner(tone, html) {
|
||||
const el = $("banner");
|
||||
if (!html) { el.classList.add("hidden"); el.innerHTML = ""; return; }
|
||||
el.className = `banner ${tone || ""}`;
|
||||
el.innerHTML = html;
|
||||
el.classList.remove("hidden");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 表单
|
||||
function buildForm() {
|
||||
const presets = $("presets");
|
||||
presets.innerHTML = "";
|
||||
for (const preset of state.schema.presets || []) {
|
||||
const btn = document.createElement("button");
|
||||
btn.type = "button";
|
||||
btn.className = "preset";
|
||||
btn.innerHTML = `${escapeHtml(preset.label)}<span class="note">${escapeHtml(preset.note || "")}</span>`;
|
||||
btn.onclick = () => { applyValues(preset.payload); $("form-error").textContent = ""; };
|
||||
presets.appendChild(btn);
|
||||
}
|
||||
|
||||
const wrap = $("form-groups");
|
||||
wrap.innerHTML = "";
|
||||
for (const group of state.schema.groups || []) {
|
||||
const box = document.createElement("fieldset");
|
||||
box.className = "group";
|
||||
const fields = (state.schema.fields || []).filter((f) => f.group === group.key);
|
||||
box.innerHTML = `<legend>${escapeHtml(group.label)}</legend>` +
|
||||
`<p class="ghint">${escapeHtml(group.hint || "")}</p>`;
|
||||
for (const field of fields) box.appendChild(buildField(field));
|
||||
wrap.appendChild(box);
|
||||
}
|
||||
applyValues({});
|
||||
}
|
||||
|
||||
function buildField(field) {
|
||||
const row = document.createElement("div");
|
||||
row.className = "field";
|
||||
row.id = `field-${field.key}`;
|
||||
const label = document.createElement("label");
|
||||
label.className = "flabel";
|
||||
label.innerHTML = `<span>${escapeHtml(field.label)}</span>` +
|
||||
(field.unit ? `<span class="unit">${escapeHtml(field.unit)}</span>` : "");
|
||||
let control;
|
||||
if (field.kind === "select") {
|
||||
control = document.createElement("select");
|
||||
for (const choice of field.choices || []) {
|
||||
const option = document.createElement("option");
|
||||
option.value = String(choice.value);
|
||||
option.textContent = choice.label;
|
||||
control.appendChild(option);
|
||||
}
|
||||
} else if (field.kind === "bool") {
|
||||
control = document.createElement("input");
|
||||
control.type = "checkbox";
|
||||
row.classList.add("check");
|
||||
} else {
|
||||
control = document.createElement("input");
|
||||
control.type = field.kind === "text" ? "text" : "number";
|
||||
if (field.kind === "number") control.step = field.step || "any";
|
||||
if (field.min !== undefined) control.min = field.min;
|
||||
if (field.max !== undefined) control.max = field.max;
|
||||
}
|
||||
control.id = `cfg-${field.key}`;
|
||||
control.dataset.key = field.key;
|
||||
control.addEventListener("input", () => { row.classList.add("dirty"); row.classList.remove("invalid"); });
|
||||
row.appendChild(label);
|
||||
row.appendChild(control);
|
||||
if (field.help) {
|
||||
const help = document.createElement("span");
|
||||
help.className = "fhelp";
|
||||
help.textContent = field.help;
|
||||
row.appendChild(help);
|
||||
}
|
||||
return row;
|
||||
}
|
||||
|
||||
function setControl(field, value) {
|
||||
const control = $(`cfg-${field.key}`);
|
||||
if (!control) return;
|
||||
if (field.kind === "bool") { control.checked = Boolean(value); return; }
|
||||
if (value === null || value === undefined) { control.value = ""; return; }
|
||||
control.value = String(value);
|
||||
}
|
||||
|
||||
function applyValues(values) {
|
||||
for (const field of state.schema.fields || []) {
|
||||
const has = Object.prototype.hasOwnProperty.call(values, field.key);
|
||||
setControl(field, has ? values[field.key] : field.default);
|
||||
const row = $(`field-${field.key}`);
|
||||
if (row) row.classList.remove("dirty", "invalid");
|
||||
}
|
||||
}
|
||||
|
||||
function collectValues() {
|
||||
const out = {};
|
||||
for (const field of state.schema.fields || []) {
|
||||
const control = $(`cfg-${field.key}`);
|
||||
if (!control) continue;
|
||||
if (field.kind === "bool") { out[field.key] = control.checked; continue; }
|
||||
const raw = String(control.value || "").trim();
|
||||
if (raw === "") { out[field.key] = null; continue; }
|
||||
if (field.kind === "integer") out[field.key] = Number.parseInt(raw, 10);
|
||||
else if (field.kind === "number") out[field.key] = Number.parseFloat(raw);
|
||||
else if (field.kind === "select") {
|
||||
const choices = field.choices || [];
|
||||
const match = choices.find((c) => String(c.value) === raw);
|
||||
out[field.key] = match ? match.value : raw;
|
||||
} else out[field.key] = raw;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/* 与后端 validate() 对齐的前置提示：只做提示，不阻止提交后端的权威判定 */
|
||||
function validateLocally(values) {
|
||||
const problems = [];
|
||||
for (const field of state.schema.fields || []) {
|
||||
const value = values[field.key];
|
||||
if (value === null || value === undefined) continue;
|
||||
if (field.kind === "integer" || field.kind === "number") {
|
||||
if (!isFinite(value)) { problems.push(`${field.label} 不是有效数字`); markInvalid(field.key, true); continue; }
|
||||
if (field.min !== undefined && value < field.min) { problems.push(`${field.label} 不能小于 ${field.min}`); markInvalid(field.key, true); continue; }
|
||||
if (field.max !== undefined && value > field.max) { problems.push(`${field.label} 不能大于 ${field.max}`); markInvalid(field.key, true); continue; }
|
||||
markInvalid(field.key, false);
|
||||
}
|
||||
}
|
||||
if (values.rollout_steps && values.timesteps && values.rollout_steps > values.timesteps) {
|
||||
problems.push("采样步数 n_steps 不能大于总步数 timesteps");
|
||||
markInvalid("rollout_steps", true);
|
||||
}
|
||||
if (values.batch_size && values.rollout_steps && values.envs
|
||||
&& values.batch_size > values.rollout_steps * values.envs) {
|
||||
problems.push("批大小 batch_size 大于 n_steps × envs，PPO 每轮数据不足以分批");
|
||||
markInvalid("batch_size", true);
|
||||
}
|
||||
return problems;
|
||||
}
|
||||
|
||||
function markInvalid(key, bad) {
|
||||
const row = $(`field-${key}`);
|
||||
if (row) row.classList.toggle("invalid", Boolean(bad));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 轮询
|
||||
async function poll() {
|
||||
if (state.polling) return;
|
||||
if (state.preview) return;
|
||||
state.polling = true;
|
||||
try {
|
||||
const query = new URLSearchParams({
|
||||
log_tail: $("log-tail").value || "400",
|
||||
metric_limit: String(METRIC_LIMIT),
|
||||
});
|
||||
if (state.metricTotal > 0) query.set("after_metric", String(state.metricTotal));
|
||||
const res = await fetch(`/api/train/state?${query.toString()}`);
|
||||
const payload = await res.json();
|
||||
if (!res.ok) throw new Error(payload.error || `HTTP ${res.status}`);
|
||||
chip("chip-conn", "连接", "已连接", "ok");
|
||||
render(payload.job);
|
||||
} catch (err) {
|
||||
chip("chip-conn", "连接", "断开", "bad");
|
||||
} finally {
|
||||
state.polling = false;
|
||||
}
|
||||
}
|
||||
|
||||
/* 合并一次快照返回的进度点。服务端用 metric_offset 给出这批点的绝对起点，
|
||||
客户端只需与"本地已吃到的下一个下标"（metricNext）对账：
|
||||
- offset === metricNext：正常续接（增量拉取），concat；
|
||||
- offset > metricNext：本地缓存与这批点之间缺了若干点（例如页面刷新后
|
||||
服务端窗口已前移），保留本地旧点会让曲线出现空洞，因此整体替换；
|
||||
- offset < metricNext：本地缓存比服务端更新（服务端重启/计数回退），
|
||||
忽略这批过期点，不动本地缓存。
|
||||
判据是"绝对下标"而不是长度：游标是服务端总数，本地可能被裁剪得更短，
|
||||
拿长度比较会把同一批点重复追加（本函数注释即为此 bug 的回归说明）。
|
||||
下标必须显式转 Number：JSON 里若出现字符串形式的数字，JS 的 >= 会退化成
|
||||
字典序比较（"14" >= "014" 为 false），静默走错分支。 */
|
||||
function absorbMetrics(job) {
|
||||
const incoming = Array.isArray(job.metrics) ? job.metrics : [];
|
||||
const rawOffset = Number(job.metric_offset);
|
||||
const offset = Number.isFinite(rawOffset) ? rawOffset : null;
|
||||
const cache = state.metricCache || [];
|
||||
if (incoming.length && offset !== null) {
|
||||
if (!cache.length || offset > state.metricNext) {
|
||||
// 首批，或与本地之间有缺口（服务端窗口前移）：整体替换，避免曲线断档
|
||||
state.metricCache = incoming.slice();
|
||||
state.metricNext = offset + incoming.length;
|
||||
} else if (offset === state.metricNext) {
|
||||
state.metricCache = cache.concat(incoming);
|
||||
state.metricNext = offset + incoming.length;
|
||||
} else {
|
||||
return; // 过期批次：本地已有更新的点，直接忽略
|
||||
}
|
||||
}
|
||||
if (state.metricCache.length > METRIC_LIMIT) {
|
||||
state.metricCache = state.metricCache.slice(state.metricCache.length - METRIC_LIMIT);
|
||||
}
|
||||
state.metrics = state.metricCache;
|
||||
const total = Number(job.metric_total);
|
||||
if (Number.isFinite(total)) state.metricTotal = total;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 渲染
|
||||
function render(job) {
|
||||
if (!job) {
|
||||
state.lastJob = null;
|
||||
renderIdle();
|
||||
return;
|
||||
}
|
||||
state.lastJob = job;
|
||||
if (job.job_id !== state.jobKey) {
|
||||
state.jobKey = job.job_id;
|
||||
state.metrics = [];
|
||||
state.metricTotal = 0;
|
||||
state.lastLogSignature = "";
|
||||
}
|
||||
absorbMetrics(job);
|
||||
|
||||
renderProcess(job);
|
||||
renderConvergence(job);
|
||||
renderCharts();
|
||||
renderLog(job);
|
||||
renderEpisodes(job);
|
||||
renderBanner(job);
|
||||
renderRecovery(job);
|
||||
renderJobLine(job);
|
||||
}
|
||||
|
||||
function renderIdle() {
|
||||
chip("chip-process", "进程", "未启动", "gray");
|
||||
chip("chip-progress", "进度", "—", "gray");
|
||||
chip("chip-elapsed", "墙钟", "—", "gray");
|
||||
chip("chip-device", "设备", "—", "gray");
|
||||
setText("process-hint", "启动任务后显示子进程与步进进度");
|
||||
setText("steps-done", "0"); setText("steps-total", "0");
|
||||
setText("progress-percent", "0.0%");
|
||||
$("bar-fill").style.width = "0%";
|
||||
setText("eta", "—"); setText("eta-clock", "—"); setText("metric-total", "0");
|
||||
for (const id of ["stat-pid", "stat-fps", "stat-elapsed", "stat-rc"]) setText(id, "—");
|
||||
setText("stat-logs", "0");
|
||||
$("timeline").innerHTML = "";
|
||||
setText("command-preview", "");
|
||||
state.lastStatus = null;
|
||||
const pauseBtn = $("btn-pause");
|
||||
pauseBtn.disabled = true;
|
||||
pauseBtn.textContent = "暂停训练";
|
||||
chip("chip-converge", "判读", "等待数据", "gray");
|
||||
$("converge-notes").innerHTML = "";
|
||||
$("metric-grid").innerHTML = '<p class="hint">尚未启动训练，暂无参数。</p>';
|
||||
setFormLocked(false);
|
||||
}
|
||||
|
||||
function renderJobLine(job) {
|
||||
const cfg = job.config || {};
|
||||
const parts = [
|
||||
`任务 ${job.job_id}`, `问题 ${cfg.problem}`, `阶段 ${cfg.stage}`,
|
||||
`口径 ${cfg.reward_profile}/${cfg.config_profile}`,
|
||||
`seed ${cfg.seed}`, `envs ${cfg.envs}·${cfg.vec_env}`,
|
||||
];
|
||||
setText("job-line", parts.join(" · "));
|
||||
chip("chip-device", "设备", `${cfg.device || "?"}`, "cyan");
|
||||
}
|
||||
|
||||
function renderProcess(job) {
|
||||
const process = job.process || {};
|
||||
const tone = process.tone || "gray";
|
||||
chip("chip-process", "进程", process.label || job.status, tone);
|
||||
chip("chip-elapsed", "墙钟", fmtDuration(process.elapsed_s), tone === "gray" ? "gray" : "cyan");
|
||||
|
||||
const percent = process.percent;
|
||||
setText("chip-progress-value", "");
|
||||
chip("chip-progress", "进度",
|
||||
percent === null || percent === undefined ? "—" : `${percent.toFixed(1)}%`,
|
||||
percent !== null && percent >= 100 ? "ok" : "cyan");
|
||||
setText("steps-done", fmtInt(process.trained_steps ?? process.done_steps));
|
||||
setText("steps-total", fmtInt(process.total_steps));
|
||||
setText("progress-percent", percent === null || percent === undefined ? "—" : `${percent.toFixed(2)}%`);
|
||||
const fill = $("bar-fill");
|
||||
fill.style.width = `${percent === null || percent === undefined ? 0 : percent}%`;
|
||||
fill.className = "bar-fill" + (tone === "bad" ? " bad" : tone === "warn" ? " warn" : tone === "ok" && percent >= 100 ? " ok" : "");
|
||||
setText("process-hint", process.hint || "");
|
||||
setText("eta", fmtDuration(process.remaining_s));
|
||||
setText("eta-clock", fmtClock(process.eta_epoch_s));
|
||||
setText("metric-total", fmtInt(process.metric_total));
|
||||
|
||||
setText("stat-pid", process.terminal ? "已退出" : (job.pid ? `PID ${job.pid}` : "—"));
|
||||
setText("stat-pgid", process.terminal ? "已回收" : "独立会话");
|
||||
const fps = state.metrics.length ? lastFinite(state.metrics, "fps") : null;
|
||||
setText("stat-fps", fps === null ? "—" : fmtNumber(fps, 1));
|
||||
setText("stat-elapsed", fmtDuration(process.elapsed_s));
|
||||
const rc = $("stat-rc");
|
||||
rc.textContent = job.returncode === null || job.returncode === undefined ? "—" : String(job.returncode);
|
||||
rc.className = "v" + (job.returncode === 0 ? " ok" : job.returncode ? " bad" : "");
|
||||
setText("stat-logs", `${fmtInt(process.log_total)} / ${fmtInt(process.log_capacity)}`);
|
||||
|
||||
renderTimeline(process);
|
||||
setText("command-preview", job.command ? `$ ${job.command.join(" ")}` : "");
|
||||
state.lastStatus = job.status;
|
||||
setFormLocked(job.status === "running" || job.status === "created"
|
||||
|| job.status === "paused" || job.status === "stopping");
|
||||
$("btn-stop").disabled = !(job.status === "running" || job.status === "created"
|
||||
|| job.status === "paused");
|
||||
const pauseBtn = $("btn-pause");
|
||||
pauseBtn.disabled = !(job.status === "running" || job.status === "paused");
|
||||
pauseBtn.textContent = job.status === "paused" ? "继续训练" : "暂停训练";
|
||||
}
|
||||
|
||||
function renderTimeline(process) {
|
||||
const wrap = $("timeline");
|
||||
const nodes = process.timeline || [];
|
||||
if (!nodes.length) { wrap.innerHTML = ""; return; }
|
||||
const active = nodes.findIndex((n) => n.state === "active");
|
||||
if (wrap.dataset.signature !== nodes.map((n) => n.label).join("|")) {
|
||||
wrap.innerHTML = "";
|
||||
nodes.forEach((node, index) => {
|
||||
if (index > 0) {
|
||||
const link = document.createElement("span");
|
||||
link.className = "link";
|
||||
link.dataset.link = String(index);
|
||||
wrap.appendChild(link);
|
||||
}
|
||||
const el = document.createElement("span");
|
||||
el.className = "node";
|
||||
el.dataset.node = node.key;
|
||||
el.innerHTML = `<span class="dot"></span><span class="txt">${escapeHtml(node.label)}</span>`;
|
||||
wrap.appendChild(el);
|
||||
});
|
||||
wrap.dataset.signature = nodes.map((n) => n.label).join("|");
|
||||
}
|
||||
nodes.forEach((node, index) => {
|
||||
const el = wrap.querySelector(`[data-node="${node.key}"]`);
|
||||
if (!el) return;
|
||||
let cls = "node";
|
||||
if (node.state === "done") cls += " done";
|
||||
else if (node.state === "active") cls += " active";
|
||||
if (node.state === "active" && process.tone === "bad") cls += " failed";
|
||||
el.className = cls;
|
||||
const link = wrap.querySelector(`[data-link="${index}"]`);
|
||||
if (link) link.className = "link" + (index <= active ? " done" : "");
|
||||
});
|
||||
}
|
||||
|
||||
function setFormLocked(locked) {
|
||||
state.saving = locked;
|
||||
for (const field of state.schema.fields || []) {
|
||||
const control = $(`cfg-${field.key}`);
|
||||
if (control) control.disabled = locked;
|
||||
}
|
||||
$("btn-start").disabled = locked;
|
||||
$("btn-start").textContent = locked ? "训练中…" : "开始训练";
|
||||
}
|
||||
|
||||
function renderBanner(job) {
|
||||
const process = job.process || {};
|
||||
if (state.preview) {
|
||||
showBanner("warn", "离线预览模式：正在渲染示例快照，<b>不会</b>启动训练进程。点「刷新」可返回实时轮询。");
|
||||
return;
|
||||
}
|
||||
if (job.status === "failed") {
|
||||
showBanner("bad", `训练异常结束（退出码 <b>${job.returncode === null ? "?" : job.returncode}</b>）：${escapeHtml(job.error || "见日志尾部")}。`);
|
||||
} else if (job.status === "paused") {
|
||||
showBanner("warn", "训练已暂停：进程与显存仍在，但停在 step 边界不再推进。点「继续训练」从暂停处恢复，或点「停止训练」保存当前模型后退出。");
|
||||
} else if (job.status === "stopped") {
|
||||
showBanner("warn", job.forced_kill
|
||||
? "停止请求超时，已强制结束进程树；最后一段模型可能未保存，请检查 checkpoint 目录。"
|
||||
: "已按请求停止训练（控制文件协作式停止），脚本已在 step 边界保存当前模型与元数据。");
|
||||
} else if (job.status === "exited") {
|
||||
showBanner("ok", "训练与评估流程正常结束，模型与元数据已写入输出目录。");
|
||||
} else if (process.remaining_s !== null && process.remaining_s !== undefined
|
||||
&& process.eta_epoch_s === null && job.status === "running") {
|
||||
showBanner("warn", `尚未产出可估算吞吐的进度点，剩余时间按墙钟上限回退估计：<b>${fmtDuration(process.remaining_s)}</b>。`);
|
||||
} else {
|
||||
showBanner("", "");
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 恢复
|
||||
function renderRecovery(job) {
|
||||
const recovery = job.recovery || {};
|
||||
const box = $("recovery-box");
|
||||
if (!box) return;
|
||||
if (!recovery.model || recovery.reason === "none") {
|
||||
box.classList.add("hidden");
|
||||
return;
|
||||
}
|
||||
box.classList.remove("hidden");
|
||||
const steps = recovery.timesteps === null || recovery.timesteps === undefined
|
||||
? "步数未知" : `${fmtInt(recovery.timesteps)} 步`;
|
||||
const when = recovery.updated_unix_s ? fmtClock(recovery.updated_unix_s) : "—";
|
||||
setText("recovery-text",
|
||||
`可恢复产物：${steps} · 来源 ${recovery.reason || "?"} · 更新 ${when} · ${recovery.model}`);
|
||||
const button = $("btn-recover");
|
||||
// 离线预览里的恢复信息是合成的，不允许真的去续训
|
||||
button.disabled = !recovery.can_resume || Boolean(state.preview);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 收敛
|
||||
function renderConvergence(job) {
|
||||
const convergence = job.convergence || {};
|
||||
chip("chip-converge", "判读", convergence.status || "等待数据", convergence.tone || "gray");
|
||||
const notes = convergence.notes || [];
|
||||
$("converge-notes").innerHTML = notes.length
|
||||
? notes.map((note) => `<li>${escapeHtml(note)}</li>`).join("")
|
||||
: "<li>暂无判读说明。</li>";
|
||||
|
||||
const grid = $("metric-grid");
|
||||
grid.innerHTML = "";
|
||||
const catalog = state.schema.metrics || [];
|
||||
for (const metric of catalog) {
|
||||
const points = seriesOf(metric.key);
|
||||
const values = points.map((p) => p[1]);
|
||||
const value = values.length ? values[values.length - 1] : null;
|
||||
const prev = values.length > 1 ? values[values.length - 2] : null;
|
||||
const delta = value !== null && prev !== null ? value - prev : null;
|
||||
const card = document.createElement("div");
|
||||
card.className = "metric-card";
|
||||
const alert = isAlert(metric.key, value);
|
||||
if (alert) card.classList.add("alert");
|
||||
const trendClass = delta === null || Math.abs(delta) < 1e-12 ? "flat"
|
||||
: delta > 0 ? "up" : "down";
|
||||
const arrow = delta === null || Math.abs(delta) < 1e-12 ? "→" : delta > 0 ? "▲" : "▼";
|
||||
card.innerHTML =
|
||||
`<div class="mhead"><span class="mname">${escapeHtml(metric.label)}</span>` +
|
||||
`<span class="mvalue">${escapeHtml(fmtValue(metric, value))}</span></div>` +
|
||||
`<canvas class="spark" data-spark="${metric.key}"></canvas>` +
|
||||
`<div class="mfoot"><span class="trend ${trendClass}" data-delta="${metric.key}">${arrow} ${escapeHtml(delta === null ? "—" : fmtValue(metric, delta))}</span>` +
|
||||
`<span>${points.length} 点</span></div>`;
|
||||
if (alert) card.title = alert;
|
||||
grid.appendChild(card);
|
||||
const canvas = card.querySelector("canvas");
|
||||
if (canvas && values.length >= 2) {
|
||||
window.Charts.sparkline(canvas, values.slice(-120), { color: metric.color });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function isAlert(key, value) {
|
||||
if (value === null || value === undefined) return null;
|
||||
if (key === "approx_kl" && value > 0.05) return "近似 KL 超过 0.05：策略更新幅度过大";
|
||||
if (key === "explained_variance" && value < 0) return "解释方差为负：价值函数尚未拟合";
|
||||
if (key === "entropy_loss" && Math.abs(value) < 1e-3) return "策略熵接近 0：探索可能过早停止";
|
||||
return null;
|
||||
}
|
||||
|
||||
function fmtValue(metric, value) {
|
||||
if (value === null || value === undefined || !isFinite(value)) return "—";
|
||||
if (metric.format === "integer") return fmtInt(value);
|
||||
return fmtNumber(value);
|
||||
}
|
||||
|
||||
function seriesOf(key) {
|
||||
const points = [];
|
||||
for (const row of state.metrics) {
|
||||
const value = Number(row[key]);
|
||||
if (!isFinite(value)) continue;
|
||||
const x = Number(row.timesteps ?? row.total_timesteps ?? points.length + 1);
|
||||
points.push([isFinite(x) ? x : points.length + 1, value]);
|
||||
}
|
||||
return points;
|
||||
}
|
||||
function lastFinite(rows, key) {
|
||||
for (let i = rows.length - 1; i >= 0; i--) {
|
||||
const value = Number(rows[i][key]);
|
||||
if (isFinite(value)) return value;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 图表
|
||||
function renderCharts() {
|
||||
const catalog = state.schema.metrics || [];
|
||||
const tabs = $("chart-tabs");
|
||||
if (tabs.dataset.signature !== catalog.map((m) => m.key).join("|")) {
|
||||
tabs.innerHTML = "";
|
||||
for (const metric of catalog) {
|
||||
const btn = document.createElement("button");
|
||||
btn.type = "button";
|
||||
btn.className = "chart-tab";
|
||||
btn.dataset.key = metric.key;
|
||||
btn.textContent = metric.label;
|
||||
btn.onclick = () => { state.chartKey = metric.key; renderCharts(); };
|
||||
tabs.appendChild(btn);
|
||||
}
|
||||
tabs.dataset.signature = catalog.map((m) => m.key).join("|");
|
||||
}
|
||||
for (const btn of tabs.querySelectorAll(".chart-tab")) {
|
||||
btn.classList.toggle("active", btn.dataset.key === state.chartKey);
|
||||
}
|
||||
|
||||
const windowSize = Number($("chart-window").value || 0);
|
||||
const primary = catalog.find((m) => m.role === "primary") || { key: "ep_rew_mean", label: "奖励", color: "#4f8cff" };
|
||||
const rewardPoints = slice(seriesOf(primary.key), windowSize);
|
||||
const layout = window.Charts.line($("chart-reward"), {
|
||||
series: [{ name: primary.label, points: rewardPoints, color: primary.color, fill: true }],
|
||||
xFormat: (v) => `${fmtInt(v)}步`,
|
||||
emptyText: "等待首个 rollout 的 ep_rew_mean……",
|
||||
});
|
||||
setText("chart-title", primary.label);
|
||||
setText("chart-sub", describeSeries(rewardPoints, primary));
|
||||
|
||||
const metric = catalog.find((m) => m.key === state.chartKey) || primary;
|
||||
const metricPoints = slice(seriesOf(metric.key), windowSize);
|
||||
const refs = [];
|
||||
if (metric.key === "approx_kl") refs.push({ value: 0.05, color: "#f87171", label: "过猛阈值 0.05" });
|
||||
if (metric.key === "explained_variance") refs.push({ value: 0, color: "#93a1bd", label: "0（不如均值预测）" });
|
||||
const layout2 = window.Charts.line($("chart-metric"), {
|
||||
series: [{ name: metric.label, points: metricPoints, color: metric.color, fill: true }],
|
||||
refs: refs,
|
||||
xFormat: (v) => `${fmtInt(v)}步`,
|
||||
emptyText: `等待 ${metric.key} 数据……`,
|
||||
});
|
||||
setText("chart2-title", metric.label);
|
||||
setText("chart2-sub", describeSeries(metricPoints, metric));
|
||||
state.layouts = { reward: layout, metric: layout2, metricKey: metric.key, primary: primary };
|
||||
}
|
||||
|
||||
function slice(points, size) {
|
||||
if (!size || points.length <= size) return points;
|
||||
return points.slice(points.length - size);
|
||||
}
|
||||
|
||||
function describeSeries(points, metric) {
|
||||
if (!points.length) return "（无数据）";
|
||||
if (points.length === 1) return `仅 1 点 · 最新 ${fmtValue(metric, points[0][1])}`;
|
||||
const first = points[0], last = points[points.length - 1];
|
||||
const delta = last[1] - first[1];
|
||||
const sign = delta > 0 ? "+" : "";
|
||||
return `${points.length} 点 · 窗口内 ${sign}${fmtValue(metric, delta)} · ` +
|
||||
`最新 ${fmtValue(metric, last[1])} @ ${fmtInt(last[0])} 步`;
|
||||
}
|
||||
|
||||
function bindChartInteractions() {
|
||||
for (const id of ["chart-reward", "chart-metric"]) {
|
||||
const canvas = $(id);
|
||||
if (!canvas) continue;
|
||||
const overlay = document.createElement("canvas");
|
||||
overlay.className = "chart-overlay";
|
||||
overlay.style.width = "100%"; overlay.style.height = "100%";
|
||||
if (canvas.parentElement) {
|
||||
canvas.parentElement.style.position = "relative";
|
||||
canvas.parentElement.appendChild(overlay);
|
||||
}
|
||||
const move = (event) => {
|
||||
const show = $("chart-cursor").checked;
|
||||
const rect = canvas.getBoundingClientRect();
|
||||
const which = id === "chart-reward" ? "reward" : "metric";
|
||||
const layout = state.layouts && state.layouts[which];
|
||||
const ctx = overlay.getContext("2d");
|
||||
const dpr = window.devicePixelRatio || 1;
|
||||
if (overlay.width !== Math.round(rect.width * dpr) || overlay.height !== Math.round(rect.height * dpr)) {
|
||||
overlay.width = Math.round(rect.width * dpr);
|
||||
overlay.height = Math.round(rect.height * dpr);
|
||||
}
|
||||
ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
|
||||
ctx.clearRect(0, 0, rect.width, rect.height);
|
||||
if (!layout || !show || !isFinite(layout.bounds.x0)) return;
|
||||
window.Charts.drawCursor(overlay, layout, {
|
||||
mouse: { x: event.clientX - rect.left, y: event.clientY - rect.top },
|
||||
xFormat: (v) => `${fmtInt(v)}步`,
|
||||
});
|
||||
};
|
||||
canvas.addEventListener("mousemove", move);
|
||||
canvas.addEventListener("mouseleave", () => {
|
||||
const ctx = overlay.getContext("2d");
|
||||
ctx.clearRect(0, 0, overlay.width, overlay.height);
|
||||
});
|
||||
}
|
||||
$("chart-window").addEventListener("change", renderCharts);
|
||||
$("chart-cursor").addEventListener("change", renderCharts);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 日志
|
||||
function renderLog(job) {
|
||||
const lines = job.log_lines || [];
|
||||
const filter = ($("log-filter").value || "").trim().toLowerCase();
|
||||
const showJson = $("log-json").checked;
|
||||
const errorsOnly = $("log-errors-only").checked;
|
||||
const view = $("log-view");
|
||||
const signature = `${lines.length}|${job.log_total}|${filter}|${showJson}|${errorsOnly}|${lines[lines.length - 1] || ""}`;
|
||||
if (signature === state.lastLogSignature) return;
|
||||
state.lastLogSignature = signature;
|
||||
|
||||
const follow = $("log-follow").checked;
|
||||
const atBottom = view.scrollTop + view.clientHeight >= view.scrollHeight - 24;
|
||||
const keep = follow || atBottom;
|
||||
|
||||
const html = [];
|
||||
let shown = 0;
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const raw = lines[i];
|
||||
const lower = raw.toLowerCase();
|
||||
if (filter && !lower.includes(filter)) continue;
|
||||
if (errorsOnly && !/error|traceback|failed|exception|错误|失败/.test(lower)) continue;
|
||||
if (!showJson && raw.trim().startsWith("{") && raw.includes('"event"')) continue;
|
||||
shown++;
|
||||
const cls = classify(raw);
|
||||
html.push(`<span class="l ${cls}">${highlight(raw, filter)}</span>`);
|
||||
}
|
||||
view.innerHTML = html.length ? html.join("") : '<span class="l dim">（无匹配日志行）</span>';
|
||||
if (keep) view.scrollTop = view.scrollHeight;
|
||||
setText("log-shown", `显示 ${shown} / 共 ${fmtInt(job.log_total)} 行`);
|
||||
setText("log-meta", `stdout/stderr 合并捕获 · 服务端保留最近 ${fmtInt(job.process ? job.process.log_capacity : 4000)} 行`);
|
||||
}
|
||||
|
||||
function classify(line) {
|
||||
const text = line.toLowerCase();
|
||||
if (/error|traceback|exception|failed|错误|失败/.test(text)) return "err";
|
||||
if (/warn|deprecat|警告/.test(text)) return "warn";
|
||||
if (line.trim().startsWith("{")) return "json";
|
||||
if (/^\s*[-=]{3,}|已保存|success|complete|评估|metadata/.test(text)) return "meta";
|
||||
if (!line.trim()) return "dim";
|
||||
return "";
|
||||
}
|
||||
|
||||
function highlight(line, filter) {
|
||||
const escaped = escapeHtml(line);
|
||||
if (!filter) return escaped;
|
||||
try {
|
||||
return escaped.replace(new RegExp(`(${filter.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")})`, "gi"),
|
||||
"<mark>$1</mark>");
|
||||
} catch (err) { return escaped; }
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 逐回合
|
||||
function renderEpisodes(job) {
|
||||
const rows = job.monitor || [];
|
||||
const tbody = $("episode-table").querySelector("tbody");
|
||||
const tail = rows.slice(-200).reverse();
|
||||
tbody.innerHTML = tail.map((row, index) => {
|
||||
const seq = rows.length - index;
|
||||
return `<tr><td>${seq}</td><td>${fmtNumber(row.r)}</td>` +
|
||||
`<td>${fmtNumber(row.l, 0)}</td><td>${fmtNumber(row.t, 3)}</td></tr>`;
|
||||
}).join("") || '<tr><td colspan="4" class="hint">尚无逐回合记录（Monitor CSV 随训练写入）</td></tr>';
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 交互
|
||||
async function submitPayload(payload) {
|
||||
const err = $("form-error");
|
||||
err.textContent = "";
|
||||
if (state.saving) return false; // 训练中表单锁定，续训按钮同样不许重复提交
|
||||
if (payload.resume && !payload.model) {
|
||||
err.textContent = "续训需要填写模型输出路径";
|
||||
return false;
|
||||
}
|
||||
const res = await fetch("/api/train/start", {
|
||||
method: "POST", headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
const body = await res.json().catch(() => ({}));
|
||||
if (!res.ok) {
|
||||
err.textContent = body.error || `启动失败（HTTP ${res.status}）`;
|
||||
return false;
|
||||
}
|
||||
state.preview = false;
|
||||
state.jobKey = null;
|
||||
state.metrics = [];
|
||||
state.metricCache = [];
|
||||
state.metricNext = 0;
|
||||
state.metricTotal = 0;
|
||||
await poll();
|
||||
return true;
|
||||
}
|
||||
|
||||
$("config-form").addEventListener("submit", async (event) => {
|
||||
event.preventDefault();
|
||||
if (state.saving) return;
|
||||
const err = $("form-error");
|
||||
err.textContent = "";
|
||||
const values = collectValues();
|
||||
const problems = validateLocally(values);
|
||||
if (problems.length) { err.textContent = problems[0]; return; }
|
||||
const payload = {};
|
||||
for (const [key, value] of Object.entries(values)) {
|
||||
if (value !== null) payload[key] = value;
|
||||
}
|
||||
await submitPayload(payload);
|
||||
});
|
||||
|
||||
$("btn-recover").addEventListener("click", async () => {
|
||||
const recovery = (state.lastJob && state.lastJob.recovery) || {};
|
||||
if (!recovery.can_resume || !recovery.model) return;
|
||||
if (!confirm(`用最近的可恢复产物续训？\n${recovery.model}\n`
|
||||
+ "当前表单里的 timesteps 会作为**追加**步数。")) return;
|
||||
// 同步回表单，用户随后可微调步数再手动点「开始训练」
|
||||
const modelField = $("cfg-model");
|
||||
if (modelField) modelField.value = recovery.model;
|
||||
const resumeField = $("cfg-resume");
|
||||
if (resumeField) resumeField.checked = true;
|
||||
const values = collectValues();
|
||||
values.model = recovery.model;
|
||||
values.resume = true;
|
||||
values.force = false; // --resume 与 --force 互斥
|
||||
await submitPayload(values);
|
||||
});
|
||||
|
||||
$("btn-pause").addEventListener("click", async () => {
|
||||
const resuming = state.lastStatus === "paused";
|
||||
const res = await fetch(resuming ? "/api/train/resume" : "/api/train/pause", {
|
||||
method: "POST", headers: { "Content-Type": "application/json" }, body: "{}",
|
||||
});
|
||||
const body = await res.json().catch(() => ({}));
|
||||
if (!res.ok) {
|
||||
showBanner("bad", `${resuming ? "继续" : "暂停"}失败：${escapeHtml(body.error || res.status)}`);
|
||||
} else if (!resuming && body.paused === false) {
|
||||
showBanner("warn", "当前没有可暂停的训练进程（可能已结束）。");
|
||||
}
|
||||
await poll();
|
||||
});
|
||||
|
||||
$("btn-stop").addEventListener("click", async () => {
|
||||
if (!confirm("确认停止当前训练？\n将通过控制文件请求优雅停止，脚本先保存当前模型与元数据再退出；"
|
||||
+ "若超时未响应则强制结束进程树。")) return;
|
||||
const res = await fetch("/api/train/stop", {
|
||||
method: "POST", headers: { "Content-Type": "application/json" }, body: "{}",
|
||||
});
|
||||
const body = await res.json().catch(() => ({}));
|
||||
if (!res.ok) showBanner("bad", `停止失败：${escapeHtml(body.error || res.status)}`);
|
||||
await poll();
|
||||
});
|
||||
|
||||
$("btn-refresh").addEventListener("click", async () => {
|
||||
state.preview = false;
|
||||
showBanner("", "");
|
||||
await poll();
|
||||
});
|
||||
|
||||
$("btn-preview").addEventListener("click", async () => {
|
||||
try {
|
||||
const scenario = $("preview-scenario") ? $("preview-scenario").value : "rising";
|
||||
const res = await fetch(`/api/train/state?preview=1&scenario=${encodeURIComponent(scenario)}&log_tail=400`);
|
||||
const payload = await res.json();
|
||||
if (!payload.job) throw new Error(payload.error || "服务端未返回示例快照");
|
||||
state.preview = true;
|
||||
state.jobKey = null;
|
||||
state.metrics = [];
|
||||
state.metricCache = [];
|
||||
state.metricNext = 0;
|
||||
state.metricTotal = 0;
|
||||
render(payload.job);
|
||||
} catch (err) {
|
||||
showBanner("bad", `离线预览不可用：${escapeHtml(err.message)}`);
|
||||
}
|
||||
});
|
||||
|
||||
$("btn-reset").addEventListener("click", () => {
|
||||
applyValues({});
|
||||
$("form-error").textContent = "";
|
||||
});
|
||||
|
||||
for (const id of ["log-filter", "log-json", "log-errors-only", "log-tail"]) {
|
||||
$(id).addEventListener("input", () => {
|
||||
state.lastLogSignature = "";
|
||||
poll();
|
||||
});
|
||||
}
|
||||
|
||||
boot();
|
||||
@@ -1,255 +0,0 @@
|
||||
/* 训练前端图表：原生 canvas 折线图，无外部依赖（离线可用，与评测前端同风格）。
|
||||
只做三件事：坐标轴自适应、多序列折线、悬停读数；不引入缩放/图例等交互，
|
||||
因为训练曲线每 2 秒整体重绘一次，重量级图表库在这里只会增加风险。
|
||||
|
||||
对外接口（挂到 globalThis.Charts）：
|
||||
Charts.fit(canvas) 按 DPR 调整画布像素
|
||||
Charts.line(canvas, {series, refs, ...}) 折线图
|
||||
Charts.sparkline(canvas, values, opts) 指标卡片内的小趋势线
|
||||
数据约定：series = [{ points: [[x, y], ...], color, width, fill }] */
|
||||
(function (global) {
|
||||
"use strict";
|
||||
|
||||
var AXIS = "#2a3550", GRID = "rgba(42,53,80,.55)", DIM = "#93a1bd", TEXT = "#e6ecf8";
|
||||
var FONT = '11px ui-monospace, SFMono-Regular, Menlo, monospace';
|
||||
|
||||
function fit(canvas) {
|
||||
var dpr = global.devicePixelRatio || 1;
|
||||
var rect = canvas.getBoundingClientRect ? canvas.getBoundingClientRect() : null;
|
||||
var cssWidth = (rect && rect.width) || canvas.clientWidth || canvas.width || 640;
|
||||
var cssHeight = (rect && rect.height) || canvas.clientHeight || canvas.height || 200;
|
||||
var w = Math.max(160, Math.round(cssWidth * dpr));
|
||||
var h = Math.max(80, Math.round(cssHeight * dpr));
|
||||
if (canvas.width !== w || canvas.height !== h) { canvas.width = w; canvas.height = h; }
|
||||
var ctx = canvas.getContext("2d");
|
||||
ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
|
||||
return { ctx: ctx, w: w / dpr, h: h / dpr };
|
||||
}
|
||||
|
||||
/* 第 0 层：数值格式（科学计数法只用于真正小而大的量级） */
|
||||
function fmt(value) {
|
||||
if (value === null || value === undefined || Number.isNaN(value)) return "—";
|
||||
var abs = Math.abs(value);
|
||||
if (abs !== 0 && (abs < 1e-3 || abs >= 1e5)) return value.toExponential(2);
|
||||
if (abs >= 100) return value.toFixed(1);
|
||||
if (abs >= 1) return value.toFixed(2);
|
||||
if (abs >= 0.01) return value.toFixed(3);
|
||||
return value.toFixed(4);
|
||||
}
|
||||
|
||||
function niceTicks(min, max, count) {
|
||||
if (!isFinite(min) || !isFinite(max)) return [0, 1];
|
||||
if (min === max) { var pad = Math.abs(min) > 1e-9 ? Math.abs(min) * 0.1 : 0.5;
|
||||
min -= pad; max += pad; }
|
||||
var span = max - min;
|
||||
var raw = span / Math.max(1, count);
|
||||
var mag = Math.pow(10, Math.floor(Math.log10(raw)));
|
||||
var norm = raw / mag;
|
||||
var step = (norm <= 1 ? 1 : norm <= 2 ? 2 : norm <= 5 ? 5 : 10) * mag;
|
||||
var start = Math.ceil(min / step) * step;
|
||||
var ticks = [];
|
||||
for (var v = start; v <= max + step * 1e-6 && ticks.length < 40; v += step) {
|
||||
ticks.push(Math.abs(v) < step * 1e-9 ? 0 : v);
|
||||
}
|
||||
if (!ticks.length) ticks = [min, max];
|
||||
return ticks;
|
||||
}
|
||||
|
||||
function bounds(series) {
|
||||
var xs = [], ys = [];
|
||||
for (var i = 0; i < series.length; i++) {
|
||||
var pts = series[i].points || [];
|
||||
for (var j = 0; j < pts.length; j++) {
|
||||
var p = pts[j];
|
||||
if (!p || !isFinite(p[0]) || !isFinite(p[1])) continue;
|
||||
xs.push(p[0]); ys.push(p[1]);
|
||||
}
|
||||
}
|
||||
if (!xs.length) return null;
|
||||
var x0 = Math.min.apply(null, xs), x1 = Math.max.apply(null, xs);
|
||||
var y0 = Math.min.apply(null, ys), y1 = Math.max.apply(null, ys);
|
||||
if (x0 === x1) { x0 -= 0.5; x1 += 0.5; }
|
||||
if (y0 === y1) { var pad = Math.abs(y0) > 1e-9 ? Math.abs(y0) * 0.08 : 0.5;
|
||||
y0 -= pad; y1 += pad; }
|
||||
return { x0: x0, x1: x1, y0: y0, y1: y1 };
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------ 折线图 */
|
||||
function line(canvas, options) {
|
||||
var opts = options || {};
|
||||
var axes = fit(canvas), ctx = axes.ctx, W = axes.w, H = axes.h;
|
||||
var padL = opts.padLeft || 62, padR = 14, padT = 10, padB = 24;
|
||||
ctx.clearRect(0, 0, W, H);
|
||||
|
||||
var series = (opts.series || []).filter(function (s) {
|
||||
return s && s.points && s.points.length;
|
||||
});
|
||||
var box = bounds(series);
|
||||
if (!box) {
|
||||
ctx.fillStyle = DIM; ctx.font = FONT;
|
||||
ctx.fillText(opts.emptyText || "等待数据……", padL, H / 2);
|
||||
return null;
|
||||
}
|
||||
var yTicks = niceTicks(box.y0, box.y1, 4);
|
||||
var yMin = Math.min(box.y0, yTicks[0]), yMax = Math.max(box.y1, yTicks[yTicks.length - 1]);
|
||||
var plotW = Math.max(10, W - padL - padR), plotH = Math.max(10, H - padT - padB);
|
||||
var sx = function (x) { return padL + (x - box.x0) / (box.x1 - box.x0) * plotW; };
|
||||
var sy = function (y) { return padT + (1 - (y - yMin) / (yMax - yMin)) * plotH; };
|
||||
|
||||
// 网格与刻度
|
||||
ctx.font = FONT; ctx.lineWidth = 1;
|
||||
for (var i = 0; i < yTicks.length; i++) {
|
||||
var y = sy(yTicks[i]);
|
||||
ctx.strokeStyle = GRID; ctx.beginPath();
|
||||
ctx.moveTo(padL, y + 0.5); ctx.lineTo(padL + plotW, y + 0.5); ctx.stroke();
|
||||
ctx.fillStyle = DIM; ctx.textAlign = "right";
|
||||
ctx.fillText(fmt(yTicks[i]), padL - 6, y + 3.5);
|
||||
}
|
||||
ctx.textAlign = "center";
|
||||
for (var t = 0; t <= 4; t++) {
|
||||
var xv = box.x0 + (box.x1 - box.x0) * (t / 4);
|
||||
var x = sx(xv);
|
||||
ctx.fillStyle = DIM;
|
||||
ctx.fillText(opts.xFormat ? opts.xFormat(xv) : fmt(xv), x, H - 7);
|
||||
}
|
||||
ctx.textAlign = "left";
|
||||
ctx.strokeStyle = AXIS; ctx.beginPath();
|
||||
ctx.moveTo(padL + 0.5, padT); ctx.lineTo(padL + 0.5, padT + plotH);
|
||||
ctx.lineTo(padL + plotW, padT + plotH); ctx.stroke();
|
||||
|
||||
// 参考线（例如 KL 的 0.05 阈值、解释方差的 0）
|
||||
var refs = opts.refs || [];
|
||||
for (var r = 0; r < refs.length; r++) {
|
||||
var ref = refs[r];
|
||||
if (ref.value === undefined || ref.value < yMin || ref.value > yMax) continue;
|
||||
var ry = sy(ref.value);
|
||||
ctx.save();
|
||||
ctx.setLineDash([4, 4]); ctx.strokeStyle = ref.color || "#f59e0b";
|
||||
ctx.beginPath(); ctx.moveTo(padL, ry); ctx.lineTo(padL + plotW, ry); ctx.stroke();
|
||||
ctx.restore();
|
||||
if (ref.label) {
|
||||
ctx.fillStyle = ref.color || "#f59e0b"; ctx.font = FONT;
|
||||
ctx.fillText(ref.label, padL + 6, ry - 4);
|
||||
}
|
||||
}
|
||||
|
||||
// 序列
|
||||
for (var s = 0; s < series.length; s++) {
|
||||
var item = series[s];
|
||||
var pts = item.points;
|
||||
ctx.strokeStyle = item.color || "#4f8cff";
|
||||
ctx.lineWidth = item.width || 1.6;
|
||||
ctx.lineJoin = "round";
|
||||
ctx.beginPath();
|
||||
for (var k = 0; k < pts.length; k++) {
|
||||
var px = sx(pts[k][0]), py = sy(pts[k][1]);
|
||||
if (k === 0) ctx.moveTo(px, py); else ctx.lineTo(px, py);
|
||||
}
|
||||
ctx.stroke();
|
||||
if (item.fill && pts.length > 1) {
|
||||
ctx.save();
|
||||
ctx.globalAlpha = 0.12; ctx.fillStyle = item.color || "#4f8cff";
|
||||
ctx.lineTo(sx(pts[pts.length - 1][0]), padT + plotH);
|
||||
ctx.lineTo(sx(pts[0][0]), padT + plotH);
|
||||
ctx.closePath(); ctx.fill(); ctx.restore();
|
||||
}
|
||||
// 末点高亮：一眼看到最新值
|
||||
var last = pts[pts.length - 1];
|
||||
ctx.fillStyle = item.color || "#4f8cff";
|
||||
ctx.beginPath(); ctx.arc(sx(last[0]), sy(last[1]), 2.6, 0, Math.PI * 2); ctx.fill();
|
||||
}
|
||||
|
||||
return { bounds: box, sx: sx, sy: sy, padL: padL, padT: padT,
|
||||
plotW: plotW, plotH: plotH, yMin: yMin, yMax: yMax, series: series };
|
||||
}
|
||||
|
||||
/* 悬停读数：把鼠标 x 映射到最接近的数据点，返回一次绘制的覆盖层信息 */
|
||||
function drawCursor(canvas, layout, options) {
|
||||
var opts = options || {};
|
||||
if (!layout || !layout.series || !layout.series.length) return;
|
||||
var axes = fit(canvas), ctx = axes.ctx;
|
||||
var mouse = opts.mouse;
|
||||
if (!mouse || mouse.x < layout.padL || mouse.x > layout.padL + layout.plotW) return;
|
||||
var target = layout.bounds.x0 +
|
||||
(mouse.x - layout.padL) / layout.plotW * (layout.bounds.x1 - layout.bounds.x0);
|
||||
var lines = [], snapX = target;
|
||||
for (var s = 0; s < layout.series.length; s++) {
|
||||
var pts = layout.series[s].points;
|
||||
if (!pts.length) continue;
|
||||
// 数据已按 x 递增，二分定位最近点
|
||||
var lo = 0, hi = pts.length - 1, best = pts[hi];
|
||||
while (lo <= hi) {
|
||||
var mid = (lo + hi) >> 1;
|
||||
if (pts[mid][0] === target) { best = pts[mid]; break; }
|
||||
if (pts[mid][0] < target) { best = pts[mid]; lo = mid + 1; }
|
||||
else { hi = mid - 1; }
|
||||
}
|
||||
var next = pts[Math.min(pts.length - 1, lo)];
|
||||
if (next && Math.abs(next[0] - target) < Math.abs(best[0] - target)) best = next;
|
||||
snapX = best[0];
|
||||
lines.push({ label: layout.series[s].name || "", color: layout.series[s].color,
|
||||
value: best[1] });
|
||||
}
|
||||
var cx = layout.sx(snapX);
|
||||
ctx.save();
|
||||
ctx.setLineDash([3, 3]); ctx.strokeStyle = "rgba(230,236,248,.35)";
|
||||
ctx.beginPath(); ctx.moveTo(cx, layout.padT);
|
||||
ctx.lineTo(cx, layout.padT + layout.plotH); ctx.stroke();
|
||||
ctx.restore();
|
||||
for (var i = 0; i < lines.length; i++) {
|
||||
var item = lines[i];
|
||||
var cy = layout.sy(item.value);
|
||||
ctx.fillStyle = item.color || "#4f8cff";
|
||||
ctx.beginPath(); ctx.arc(cx, cy, 3.2, 0, Math.PI * 2); ctx.fill();
|
||||
ctx.strokeStyle = "rgba(11,15,26,.9)"; ctx.lineWidth = 1.4; ctx.stroke();
|
||||
}
|
||||
var text = (opts.xFormat ? opts.xFormat(snapX) : fmt(snapX)) + " · " +
|
||||
lines.map(function (item) { return (item.label ? item.label + " " : "") + fmt(item.value); })
|
||||
.join(" ");
|
||||
ctx.font = FONT;
|
||||
var tw = ctx.measureText(text).width + 14;
|
||||
var bx = Math.min(cx + 8, layout.padL + layout.plotW - tw);
|
||||
bx = Math.max(layout.padL + 2, bx);
|
||||
var by = layout.padT + 4;
|
||||
ctx.fillStyle = "rgba(23,30,46,.94)"; ctx.strokeStyle = AXIS;
|
||||
ctx.beginPath();
|
||||
if (ctx.roundRect) { ctx.roundRect(bx, by, tw, 20, 6); ctx.fill(); ctx.stroke(); }
|
||||
else { ctx.fillRect(bx, by, tw, 20); ctx.strokeRect(bx, by, tw, 20); }
|
||||
ctx.fillStyle = TEXT; ctx.textAlign = "left";
|
||||
ctx.fillText(text, bx + 7, by + 14);
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------ 迷你趋势线 */
|
||||
function sparkline(canvas, values, options) {
|
||||
var opts = options || {};
|
||||
var axes = fit(canvas), ctx = axes.ctx, W = axes.w, H = axes.h;
|
||||
ctx.clearRect(0, 0, W, H);
|
||||
var clean = (values || []).filter(function (v) { return isFinite(v); });
|
||||
if (clean.length < 2) {
|
||||
ctx.strokeStyle = GRID; ctx.beginPath();
|
||||
ctx.moveTo(0, H - 1.5); ctx.lineTo(W, H - 1.5); ctx.stroke();
|
||||
return;
|
||||
}
|
||||
var lo = Math.min.apply(null, clean), hi = Math.max.apply(null, clean);
|
||||
if (lo === hi) { lo -= 0.5; hi += 0.5; }
|
||||
var pad = 2;
|
||||
var X = function (i) { return (i / (clean.length - 1)) * W; };
|
||||
var Y = function (v) { return H - pad - (v - lo) / (hi - lo) * (H - pad * 2); };
|
||||
ctx.strokeStyle = opts.color || "#4f8cff"; ctx.lineWidth = 1.4;
|
||||
ctx.beginPath();
|
||||
for (var i = 0; i < clean.length; i++) {
|
||||
if (i === 0) ctx.moveTo(X(i), Y(clean[i])); else ctx.lineTo(X(i), Y(clean[i]));
|
||||
}
|
||||
ctx.stroke();
|
||||
if (opts.fill !== false) {
|
||||
ctx.save(); ctx.globalAlpha = 0.14; ctx.fillStyle = opts.color || "#4f8cff";
|
||||
ctx.lineTo(W, H); ctx.lineTo(0, H); ctx.closePath(); ctx.fill(); ctx.restore();
|
||||
}
|
||||
var lastX = X(clean.length - 1), lastY = Y(clean[clean.length - 1]);
|
||||
ctx.fillStyle = opts.color || "#4f8cff";
|
||||
ctx.beginPath(); ctx.arc(lastX - 1.5, lastY, 2.2, 0, Math.PI * 2); ctx.fill();
|
||||
}
|
||||
|
||||
global.Charts = { fit: fit, line: line, drawCursor: drawCursor,
|
||||
sparkline: sparkline, fmt: fmt };
|
||||
})(typeof globalThis !== "undefined" ? globalThis : this);
|
||||
@@ -1,180 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>模型训练前端 · CUMCM 2026 B HRL</title>
|
||||
<link rel="stylesheet" href="/style.css">
|
||||
</head>
|
||||
<body>
|
||||
<header class="topbar">
|
||||
<div class="brand">
|
||||
<h1>模型训练前端 <span class="sub">HRL 高层 MaskablePPO</span></h1>
|
||||
<p class="tagline" id="job-line">尚未启动训练任务</p>
|
||||
</div>
|
||||
<div class="topstatus">
|
||||
<span class="chip gray" id="chip-process">进程<b>未启动</b></span>
|
||||
<span class="chip gray" id="chip-progress">进度<b>—</b></span>
|
||||
<span class="chip gray" id="chip-elapsed">墙钟<b>—</b></span>
|
||||
<span class="chip gray" id="chip-device">设备<b>—</b></span>
|
||||
<span class="chip gray" id="chip-conn">连接<b>探测中</b></span>
|
||||
</div>
|
||||
<div class="topactions">
|
||||
<button id="btn-refresh" type="button" title="立即拉取一次快照">刷新</button>
|
||||
<label class="inline" title="离线预览：读取示例快照，用于无训练进程时检查界面">
|
||||
预览
|
||||
<select id="preview-scenario">
|
||||
<option value="rising">上升</option>
|
||||
<option value="plateau">平台期</option>
|
||||
<option value="kl-spike">KL 过猛</option>
|
||||
</select>
|
||||
</label>
|
||||
<button id="btn-preview" type="button" title="渲染示例快照，不启动任何训练进程">离线预览</button>
|
||||
<button id="btn-pause" type="button" disabled>暂停训练</button>
|
||||
<button id="btn-stop" class="danger" type="button" disabled>停止训练</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<div id="banner" class="banner hidden"></div>
|
||||
|
||||
<div id="recovery-box" class="recovery hidden">
|
||||
<span class="hint" id="recovery-text">—</span>
|
||||
<button id="btn-recover" type="button" title="把最近一次完整落盘的产物作为 --resume 目标继续训练">用最近检查点续训</button>
|
||||
</div>
|
||||
|
||||
<!-- =============================================== 1. 执行进度与进程 -->
|
||||
<section class="panel" id="panel-process">
|
||||
<div class="panel-head">
|
||||
<h2>执行进度与进程</h2>
|
||||
<span class="hint" id="process-hint">启动任务后显示子进程与步进进度</span>
|
||||
</div>
|
||||
|
||||
<div class="progress-row">
|
||||
<div class="progress-main">
|
||||
<div class="progress-labels">
|
||||
<span><b id="steps-done">0</b> / <span id="steps-total">0</span> 环境步</span>
|
||||
<span class="hint" id="progress-percent">0.0%</span>
|
||||
</div>
|
||||
<div class="bar"><div class="bar-fill" id="bar-fill"></div></div>
|
||||
<div class="progress-foot">
|
||||
<span class="hint">剩余估计 <b id="eta">—</b></span>
|
||||
<span class="hint">预计结束 <b id="eta-clock">—</b></span>
|
||||
<span class="hint">进度点 <b id="metric-total">0</b></span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="stat-grid">
|
||||
<div class="stat"><span class="k">子进程 PID</span><b class="v" id="stat-pid">—</b></div>
|
||||
<div class="stat"><span class="k">进程组 / 会话</span><b class="v" id="stat-pgid">独立</b></div>
|
||||
<div class="stat"><span class="k">吞吐 FPS</span><b class="v" id="stat-fps">—</b></div>
|
||||
<div class="stat"><span class="k">已用墙钟</span><b class="v" id="stat-elapsed">—</b></div>
|
||||
<div class="stat"><span class="k">退出码</span><b class="v" id="stat-rc">—</b></div>
|
||||
<div class="stat"><span class="k">日志行数</span><b class="v" id="stat-logs">0</b></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="timeline" id="timeline"></div>
|
||||
<p class="hint" id="command-preview"></p>
|
||||
</section>
|
||||
|
||||
<!-- =============================================== 2. 训练配置 -->
|
||||
<section class="panel" id="panel-config">
|
||||
<div class="panel-head">
|
||||
<h2>训练配置</h2>
|
||||
<span class="hint">字段与后端 <code>TrainConfig</code> 同源；留空字段取默认值</span>
|
||||
</div>
|
||||
<div class="presets" id="presets"></div>
|
||||
<form id="config-form" autocomplete="off">
|
||||
<div class="form-groups" id="form-groups"></div>
|
||||
<div class="form-foot">
|
||||
<button id="btn-start" class="primary" type="submit">开始训练</button>
|
||||
<button id="btn-reset" type="button">恢复默认</button>
|
||||
<span class="form-error" id="form-error"></span>
|
||||
</div>
|
||||
</form>
|
||||
</section>
|
||||
|
||||
<!-- =============================================== 3. 参数收敛 -->
|
||||
<section class="panel" id="panel-convergence">
|
||||
<div class="panel-head">
|
||||
<h2>参数收敛状态</h2>
|
||||
<span class="hint">判据为最近窗口的斜率与波动，仅供人工复核</span>
|
||||
</div>
|
||||
<div class="converge-head">
|
||||
<span class="chip gray" id="chip-converge">判读<b>等待数据</b></span>
|
||||
<ul class="notes" id="converge-notes"></ul>
|
||||
</div>
|
||||
<div class="metric-grid" id="metric-grid"></div>
|
||||
|
||||
<div class="chart-toolbar">
|
||||
<div class="chart-tabs" id="chart-tabs"></div>
|
||||
<label class="inline">最近点数
|
||||
<select id="chart-window">
|
||||
<option value="0">全部</option>
|
||||
<option value="40">40</option>
|
||||
<option value="120" selected>120</option>
|
||||
<option value="400">400</option>
|
||||
</select>
|
||||
</label>
|
||||
<label class="inline"><input type="checkbox" id="chart-cursor" checked> 悬停读数</label>
|
||||
</div>
|
||||
<div class="chart-stack">
|
||||
<figure class="chart-figure">
|
||||
<figcaption><b id="chart-title">平均回合奖励</b><span class="hint" id="chart-sub"></span></figcaption>
|
||||
<canvas id="chart-reward" height="240"></canvas>
|
||||
</figure>
|
||||
<figure class="chart-figure">
|
||||
<figcaption><b id="chart2-title">近似 KL</b><span class="hint" id="chart2-sub"></span></figcaption>
|
||||
<canvas id="chart-metric" height="180"></canvas>
|
||||
</figure>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- =============================================== 4. 实时运行日志 -->
|
||||
<section class="panel" id="panel-log">
|
||||
<div class="panel-head">
|
||||
<h2>实时运行日志</h2>
|
||||
<span class="hint" id="log-meta">stdout/stderr 合并捕获</span>
|
||||
</div>
|
||||
<div class="log-toolbar">
|
||||
<input id="log-filter" type="search" placeholder="过滤关键字，如 rollout / progress / error">
|
||||
<label class="inline"><input type="checkbox" id="log-json" checked> 显示进度 JSON</label>
|
||||
<label class="inline"><input type="checkbox" id="log-errors-only"> 只看错误</label>
|
||||
<label class="inline"><input type="checkbox" id="log-follow" checked> 自动跟随到底部</label>
|
||||
<label class="inline">尾部行数
|
||||
<select id="log-tail">
|
||||
<option value="200">200</option>
|
||||
<option value="400" selected>400</option>
|
||||
<option value="1000">1000</option>
|
||||
<option value="2000">2000</option>
|
||||
</select>
|
||||
</label>
|
||||
<span class="hint" id="log-shown"></span>
|
||||
</div>
|
||||
<pre class="log-view" id="log-view">（暂无输出）</pre>
|
||||
</section>
|
||||
|
||||
<!-- =============================================== 5. 逐回合奖励 -->
|
||||
<section class="panel" id="panel-episodes">
|
||||
<div class="panel-head">
|
||||
<h2>逐回合记录</h2>
|
||||
<span class="hint">来自训练脚本写入的 Monitor CSV（每回合奖励/长度/时刻）</span>
|
||||
</div>
|
||||
<div class="table-wrap">
|
||||
<table id="episode-table">
|
||||
<thead><tr><th>#</th><th>奖励 r</th><th>长度 l</th><th>时刻 t (s)</th></tr></thead>
|
||||
<tbody></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</section>
|
||||
</main>
|
||||
|
||||
<footer>
|
||||
本地回环训练前端：与评测前端共用「白名单静态资源 + JSON 轮询 API」模式；
|
||||
命令由服务端以参数列表构造（不经过 shell），停止时向独立进程组发送 SIGTERM。
|
||||
</footer>
|
||||
|
||||
<script src="/charts.js"></script>
|
||||
<script src="/app.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,230 +0,0 @@
|
||||
/* 训练前端样式：沿用评测前端（referee/static/style.css）的暗色变量与圆角面板
|
||||
语言，保证两个界面像同一套系统的两个页面；本页信息密度更高：
|
||||
进度条、进程时间线、参数卡片网格、曲线与日志终端。 */
|
||||
|
||||
:root {
|
||||
--bg: #0f1420; --panel: #171e2e; --panel2: #1d2740; --line: #2a3550;
|
||||
--text: #e6ecf8; --dim: #93a1bd; --accent: #4f8cff; --ok: #34d399;
|
||||
--warn: #f59e0b; --bad: #f87171; --cyan: #38bdf8; --gray: #6b7280;
|
||||
--mono: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
body {
|
||||
margin: 0; background: var(--bg); color: var(--text);
|
||||
font: 14px/1.5 "PingFang SC", "Microsoft YaHei", system-ui, sans-serif;
|
||||
}
|
||||
code, .mono { background: var(--panel2); padding: 1px 6px; border-radius: 4px; color: var(--cyan); font-family: var(--mono); font-size: 12px; }
|
||||
main { max-width: 1240px; margin: 0 auto; padding: 16px 22px 48px; }
|
||||
footer { color: var(--dim); font-size: 12px; text-align: center; padding: 14px 22px; border-top: 1px solid var(--line); }
|
||||
.hidden { display: none !important; }
|
||||
.hint { color: var(--dim); font-size: 12px; }
|
||||
h2 { font-size: 15px; margin: 0; }
|
||||
|
||||
/* ------------------------------------------------------------ 顶栏 */
|
||||
.topbar {
|
||||
position: sticky; top: 0; z-index: 20;
|
||||
display: flex; align-items: center; gap: 18px; flex-wrap: wrap;
|
||||
padding: 12px 22px; border-bottom: 1px solid var(--line);
|
||||
background: rgba(23,30,46,.96); backdrop-filter: blur(6px);
|
||||
}
|
||||
.brand h1 { font-size: 17px; margin: 0; }
|
||||
.brand .sub { font-size: 12px; color: var(--dim); font-weight: normal; margin-left: 8px; }
|
||||
.brand .tagline { margin: 3px 0 0; color: var(--dim); font-size: 12px; font-family: var(--mono); }
|
||||
.topstatus { display: flex; gap: 8px; flex-wrap: wrap; margin-left: auto; }
|
||||
.topactions { display: flex; gap: 8px; }
|
||||
|
||||
.chip {
|
||||
background: var(--panel2); border: 1px solid var(--line); border-radius: 999px;
|
||||
padding: 3px 12px; font-size: 12px; color: var(--dim); white-space: nowrap;
|
||||
}
|
||||
.chip b { color: var(--text); font-weight: 600; margin-left: 4px; }
|
||||
.chip.gray b { color: var(--gray); }
|
||||
.chip.ok b { color: var(--ok); }
|
||||
.chip.warn b { color: var(--warn); }
|
||||
.chip.bad b { color: var(--bad); }
|
||||
.chip.cyan b { color: var(--cyan); }
|
||||
|
||||
button, .button {
|
||||
background: var(--panel2); color: var(--text); border: 1px solid var(--line);
|
||||
border-radius: 6px; padding: 7px 14px; font-size: 13px; cursor: pointer;
|
||||
}
|
||||
button:hover:enabled, .button:hover { border-color: var(--accent); }
|
||||
button:disabled { opacity: .45; cursor: not-allowed; }
|
||||
button.primary { background: var(--accent); border-color: var(--accent); color: #fff; }
|
||||
button.primary:hover:enabled { filter: brightness(1.08); }
|
||||
button.danger { background: transparent; border-color: var(--bad); color: var(--bad); }
|
||||
button.danger:hover:enabled { background: rgba(248,113,113,.12); }
|
||||
|
||||
.banner {
|
||||
border: 1px solid var(--bad); border-left: 4px solid var(--bad);
|
||||
background: rgba(248,113,113,.08); border-radius: 8px;
|
||||
padding: 10px 14px; margin: 14px 0 0; font-size: 13px;
|
||||
}
|
||||
.banner.ok { border-color: var(--ok); border-left-color: var(--ok); background: rgba(52,211,153,.08); }
|
||||
.banner.warn { border-color: var(--warn); border-left-color: var(--warn); background: rgba(245,158,11,.08); }
|
||||
.banner b { font-family: var(--mono); }
|
||||
|
||||
/* 恢复条：显示最近一份完整落盘的产物，并提供一键续训 */
|
||||
.recovery {
|
||||
display: flex; align-items: center; justify-content: space-between; gap: 12px;
|
||||
border: 1px solid var(--cyan, #22d3ee); border-left: 4px solid var(--cyan, #22d3ee);
|
||||
background: rgba(34,211,238,.07); border-radius: 8px;
|
||||
padding: 8px 12px; margin: 10px 0 0; font-size: 13px;
|
||||
}
|
||||
.recovery .hint { overflow-wrap: anywhere; }
|
||||
|
||||
/* ------------------------------------------------------------ 面板 */
|
||||
.panel {
|
||||
border: 1px solid var(--line); border-radius: 12px; background: var(--panel);
|
||||
padding: 14px 16px; margin-top: 16px;
|
||||
}
|
||||
.panel-head { display: flex; align-items: baseline; gap: 12px; margin-bottom: 12px; flex-wrap: wrap; }
|
||||
.panel-head h2 { color: var(--text); }
|
||||
.panel-head h2::before {
|
||||
content: ""; display: inline-block; width: 3px; height: 13px; margin-right: 8px;
|
||||
background: var(--accent); border-radius: 2px; vertical-align: -1px;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------ 进度 */
|
||||
.progress-row { display: flex; gap: 20px; align-items: stretch; flex-wrap: wrap; }
|
||||
.progress-main { flex: 1 1 380px; min-width: 320px; }
|
||||
.progress-labels { display: flex; justify-content: space-between; font-size: 13px; margin-bottom: 6px; }
|
||||
.progress-labels b { font-family: var(--mono); font-size: 15px; }
|
||||
.bar { height: 12px; border-radius: 999px; background: #0b0f1a; border: 1px solid var(--line); overflow: hidden; }
|
||||
.bar-fill {
|
||||
height: 100%; width: 0%; border-radius: 999px;
|
||||
background: linear-gradient(90deg, #2f6fe0, var(--accent) 60%, var(--cyan));
|
||||
transition: width .3s ease;
|
||||
}
|
||||
.bar-fill.warn { background: linear-gradient(90deg, #b45309, var(--warn)); }
|
||||
.bar-fill.bad { background: linear-gradient(90deg, #b91c1c, var(--bad)); }
|
||||
.bar-fill.ok { background: linear-gradient(90deg, #059669, var(--ok)); }
|
||||
.progress-foot { display: flex; gap: 18px; margin-top: 7px; flex-wrap: wrap; }
|
||||
|
||||
.stat-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(132px, 1fr)); gap: 8px; flex: 1 1 420px; }
|
||||
.stat {
|
||||
border: 1px solid var(--line); border-radius: 8px; background: var(--panel2);
|
||||
padding: 7px 10px; display: flex; flex-direction: column; gap: 2px;
|
||||
}
|
||||
.stat .k { color: var(--dim); font-size: 11px; }
|
||||
.stat .v { font-family: var(--mono); font-size: 15px; }
|
||||
.stat .v.ok { color: var(--ok); }
|
||||
.stat .v.warn { color: var(--warn); }
|
||||
.stat .v.bad { color: var(--bad); }
|
||||
|
||||
.timeline { display: flex; align-items: center; gap: 0; margin-top: 16px; flex-wrap: wrap; }
|
||||
.node { display: flex; align-items: center; gap: 8px; }
|
||||
.node .dot {
|
||||
width: 12px; height: 12px; border-radius: 50%; border: 2px solid var(--line);
|
||||
background: var(--panel2); flex: none;
|
||||
}
|
||||
.node .txt { font-size: 12px; color: var(--dim); }
|
||||
.node.done .dot { border-color: var(--ok); background: var(--ok); }
|
||||
.node.done .txt { color: var(--text); }
|
||||
.node.active .dot { border-color: var(--accent); background: var(--accent); box-shadow: 0 0 0 4px rgba(79,140,255,.18); }
|
||||
.node.active .txt { color: var(--accent); font-weight: 600; }
|
||||
.node.failed .dot { border-color: var(--bad); background: var(--bad); }
|
||||
.node.failed .txt { color: var(--bad); }
|
||||
.link { width: 34px; height: 2px; background: var(--line); flex: none; margin: 0 8px; }
|
||||
.link.done { background: var(--ok); }
|
||||
#command-preview { margin: 12px 0 0; font-family: var(--mono); word-break: break-all; line-height: 1.7; }
|
||||
|
||||
/* ------------------------------------------------------------ 表单 */
|
||||
.presets { display: flex; flex-wrap: wrap; gap: 8px; margin-bottom: 14px; }
|
||||
.preset {
|
||||
border: 1px solid var(--line); border-radius: 8px; background: var(--panel2);
|
||||
padding: 7px 12px; font-size: 12px; cursor: pointer; color: var(--text);
|
||||
}
|
||||
.preset:hover { border-color: var(--accent); }
|
||||
.preset .note { display: block; color: var(--dim); font-size: 11px; margin-top: 2px; }
|
||||
.form-groups { display: grid; grid-template-columns: repeat(auto-fit, minmax(268px, 1fr)); gap: 14px; }
|
||||
fieldset.group { border: 1px solid var(--line); border-radius: 10px; padding: 10px 12px 12px; margin: 0; background: #141b29; }
|
||||
fieldset.group legend { color: var(--dim); font-size: 12px; padding: 0 6px; }
|
||||
fieldset.group .ghint { color: var(--dim); font-size: 11px; margin: 0 0 8px; }
|
||||
.field { display: flex; flex-direction: column; gap: 4px; margin-bottom: 10px; }
|
||||
.field .flabel { font-size: 12px; color: var(--dim); display: flex; gap: 6px; align-items: baseline; }
|
||||
.field .flabel .unit { color: var(--gray); font-size: 11px; }
|
||||
.field input[type="text"], .field input[type="number"], .field select {
|
||||
background: var(--panel2); border: 1px solid var(--line); color: var(--text);
|
||||
border-radius: 6px; padding: 6px 9px; font-size: 13px; width: 100%;
|
||||
font-family: var(--mono);
|
||||
}
|
||||
.field input:focus, .field select:focus, .log-toolbar input:focus { outline: none; border-color: var(--accent); }
|
||||
.field .fhelp { font-size: 11px; color: var(--gray); }
|
||||
.field.check { flex-direction: row; align-items: center; gap: 8px; }
|
||||
.field.check input { width: auto; }
|
||||
.field.dirty input, .field.dirty select { border-color: rgba(79,140,255,.65); }
|
||||
.field.invalid input, .field.invalid select { border-color: var(--bad); }
|
||||
.form-foot { display: flex; align-items: center; gap: 12px; margin-top: 14px; flex-wrap: wrap; }
|
||||
.form-error { color: var(--bad); font-size: 12px; }
|
||||
|
||||
/* ------------------------------------------------------------ 收敛 */
|
||||
.converge-head { display: flex; gap: 14px; align-items: flex-start; flex-wrap: wrap; margin-bottom: 12px; }
|
||||
.notes { margin: 0; padding-left: 18px; color: var(--dim); font-size: 12px; max-width: 760px; }
|
||||
.notes li { margin: 2px 0; }
|
||||
.metric-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(186px, 1fr)); gap: 10px; }
|
||||
.metric-card {
|
||||
border: 1px solid var(--line); border-radius: 10px; background: var(--panel2);
|
||||
padding: 9px 11px; display: flex; flex-direction: column; gap: 5px;
|
||||
}
|
||||
.metric-card .mhead { display: flex; justify-content: space-between; align-items: baseline; gap: 8px; }
|
||||
.metric-card .mname { font-size: 12px; color: var(--dim); }
|
||||
.metric-card .mvalue { font-family: var(--mono); font-size: 17px; }
|
||||
.metric-card .mfoot { display: flex; justify-content: space-between; font-size: 11px; color: var(--gray); gap: 6px; }
|
||||
.metric-card canvas { width: 100%; height: 40px; display: block; }
|
||||
.trend.up { color: var(--ok); } .trend.down { color: var(--warn); }
|
||||
.trend.flat { color: var(--dim); } .trend.none { color: var(--gray); }
|
||||
.metric-card.alert { border-color: rgba(248,113,113,.55); }
|
||||
.metric-card.alert .mvalue { color: var(--bad); }
|
||||
|
||||
/* ------------------------------------------------------------ 图表 */
|
||||
.chart-toolbar { display: flex; align-items: center; gap: 14px; flex-wrap: wrap; margin: 16px 0 8px; }
|
||||
.chart-tabs { display: flex; gap: 6px; flex-wrap: wrap; }
|
||||
.chart-tab {
|
||||
border: 1px solid var(--line); background: var(--transparent, transparent);
|
||||
color: var(--dim); border-radius: 999px; padding: 4px 12px; font-size: 12px;
|
||||
}
|
||||
.chart-tab.active { border-color: var(--accent); color: var(--text); background: var(--panel2); }
|
||||
label.inline { font-size: 12px; color: var(--dim); display: flex; gap: 6px; align-items: center; }
|
||||
label.inline input[type="checkbox"] { width: auto; }
|
||||
label.inline select, .log-toolbar select {
|
||||
background: var(--panel2); color: var(--text); border: 1px solid var(--line);
|
||||
border-radius: 6px; padding: 4px 8px; font-size: 12px;
|
||||
}
|
||||
.chart-stack { display: flex; flex-direction: column; gap: 14px; }
|
||||
.chart-figure { margin: 0; border: 1px solid var(--line); border-radius: 10px; background: #0b0f1a; padding: 10px 12px 4px; }
|
||||
.chart-figure figcaption { display: flex; gap: 10px; align-items: baseline; font-size: 12px; margin-bottom: 6px; }
|
||||
.chart-figure figcaption b { color: var(--text); font-size: 13px; }
|
||||
.chart-figure canvas { width: 100%; display: block; }
|
||||
|
||||
/* ------------------------------------------------------------ 日志 */
|
||||
.log-toolbar { display: flex; align-items: center; gap: 12px; flex-wrap: wrap; margin-bottom: 8px; }
|
||||
.log-toolbar input[type="search"] {
|
||||
background: var(--panel2); border: 1px solid var(--line); color: var(--text);
|
||||
border-radius: 6px; padding: 6px 10px; font-size: 12px; min-width: 240px;
|
||||
}
|
||||
.log-view {
|
||||
margin: 0; background: #0b0f1a; border: 1px solid var(--line); border-radius: 10px;
|
||||
padding: 10px 12px; height: 320px; overflow: auto; font-family: var(--mono);
|
||||
font-size: 12px; line-height: 1.55; white-space: pre-wrap; word-break: break-word;
|
||||
}
|
||||
.log-view .l { display: block; }
|
||||
.log-view .l.err { color: var(--bad); }
|
||||
.log-view .l.warn { color: var(--warn); }
|
||||
.log-view .l.json { color: var(--cyan); }
|
||||
.log-view .l.meta { color: var(--ok); }
|
||||
.log-view .l.dim { color: var(--gray); }
|
||||
.log-view mark { background: rgba(245,158,11,.28); color: inherit; border-radius: 2px; }
|
||||
|
||||
/* ------------------------------------------------------------ 表格 */
|
||||
.table-wrap { border: 1px solid var(--line); border-radius: 10px; overflow: auto; max-height: 300px; }
|
||||
table { width: 100%; border-collapse: collapse; font-size: 12px; }
|
||||
th, td { text-align: left; padding: 6px 10px; border-bottom: 1px solid var(--line); white-space: nowrap; }
|
||||
th { position: sticky; top: 0; background: var(--panel2); color: var(--dim); font-weight: 500; }
|
||||
td { font-family: var(--mono); }
|
||||
tbody tr:hover { background: rgba(79,140,255,.06); }
|
||||
|
||||
@media (max-width: 760px) {
|
||||
.topstatus { margin-left: 0; }
|
||||
.progress-row { gap: 12px; }
|
||||
}
|
||||
Reference in New Issue
Block a user