114514
This commit is contained in:
Horldsence
+35
-5
@@ -257,19 +257,38 @@ def _run(args) -> int:
|
||||
else:
|
||||
strategy = args.strategy
|
||||
loop_cls = Q3_STRATEGIES.get(strategy)
|
||||
high_policy = None
|
||||
low_policy = None
|
||||
try:
|
||||
if strategy == "rl":
|
||||
# 惰性导入：默认运行闭环不加载 numpy/torch/stable-baselines3。
|
||||
from .planner.rl_loop import RLBayesLoop, RLDirectionalLoop
|
||||
loop_cls = RLBayesLoop if args.problem == 3 else RLDirectionalLoop
|
||||
elif strategy == "hrl":
|
||||
# P1：高层 MaskablePPO 调度。缺少模型时显式报错，不降级伪装。
|
||||
if not args.hrl_model:
|
||||
raise RuntimeError(
|
||||
"--strategy hrl/--q4-strategy hrl 需要 --hrl-model；"
|
||||
"无模型启发式请使用 benchmark_hrl.py 的 hrl_heuristic 变体。")
|
||||
from .hrl.policy import HighLevelPolicy, LowLevelPolicy
|
||||
from .planner.hrl_loop import HRLLoop
|
||||
high_policy = HighLevelPolicy(
|
||||
args.hrl_model, device=args.hrl_device, problem=args.problem)
|
||||
if args.hrl_low_model:
|
||||
low_policy = LowLevelPolicy(args.hrl_low_model, device=args.hrl_device)
|
||||
loop_cls = HRLLoop
|
||||
if loop_cls is None:
|
||||
raise RuntimeError(f"未知策略: {strategy}")
|
||||
if strategy == "rl":
|
||||
summary = loop_cls(client, config, model_path=args.rl_model).run()
|
||||
elif strategy == "hrl":
|
||||
summary = loop_cls(
|
||||
client, config, problem=args.problem, high_policy=high_policy,
|
||||
low_policy=low_policy, trace_executor=args.trace_low).run()
|
||||
else:
|
||||
summary = loop_cls(client, config).run()
|
||||
except RuntimeError as exc:
|
||||
print(f"RL 策略不可用: {exc}", file=sys.stderr)
|
||||
print(f"策略不可用: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
finally:
|
||||
if judge is not None:
|
||||
@@ -323,19 +342,30 @@ def main(argv=None) -> int:
|
||||
run.add_argument("--log", default=None, help="会话日志 JSONL 输出路径")
|
||||
run.add_argument("--problem", type=int, choices=(3, 4), default=3,
|
||||
help="问题3=全向源，问题4=混合定向源（默认 3）")
|
||||
run.add_argument("--strategy", choices=tuple(Q3_STRATEGIES) + ("rl",),
|
||||
run.add_argument("--strategy", choices=tuple(Q3_STRATEGIES) + ("rl", "hrl"),
|
||||
default="bayes",
|
||||
help="问题3策略：bayes=粒子滤波+信息增益+滚动调度（默认，"
|
||||
"自建消融相对 seven 再降 19.5%%），seven=七点批量，"
|
||||
"grid=49点网格基线，rl=PPO 强化学习选点（需先训练模型）；"
|
||||
"grid=49点网格基线，rl=PPO 强化学习选点（需先训练模型），"
|
||||
"hrl=高层 MaskablePPO 分层调度（需 --hrl-model）；"
|
||||
"仅 --problem 3 生效")
|
||||
run.add_argument("--q4-strategy", choices=tuple(Q4_STRATEGIES) + ("rl",),
|
||||
run.add_argument("--q4-strategy", choices=tuple(Q4_STRATEGIES) + ("rl", "hrl"),
|
||||
default="directional",
|
||||
help="问题4策略：directional=49点覆盖+遮挡梯子+后备清除（默认），"
|
||||
"rl=PPO 强化学习选点（需先训练模型）；仅 --problem 4 生效")
|
||||
"rl=PPO 强化学习选点（需先训练模型），"
|
||||
"hrl=高层 MaskablePPO 分层调度（需 --hrl-model）；仅 --problem 4 生效")
|
||||
run.add_argument("--rl-model", default=None,
|
||||
help="RL 模型路径（默认 output/rl/q3_ppo.zip 或 "
|
||||
"output/rl/q4_ppo.zip；训练脚本 scripts/train_rl.py）")
|
||||
run.add_argument("--hrl-model", default=None,
|
||||
help="高层 HRL MaskablePPO 模型路径（--strategy hrl 时必填；"
|
||||
"训练脚本 scripts/train_hrl_high.py）")
|
||||
run.add_argument("--hrl-low-model", default=None,
|
||||
help="可选低层 HRL 模型；当前只用于轨迹推演留痕，不改变协议终点")
|
||||
run.add_argument("--hrl-device", choices=("auto", "cpu", "cuda", "mps"),
|
||||
default="auto", help="HRL 模型推理设备")
|
||||
run.add_argument("--trace-low", action="store_true",
|
||||
help="开启低层到达推演留痕（本地虚拟轨迹，不增加协议耗时）")
|
||||
run.add_argument("--bearing-margin-deg", type=float, default=0.005,
|
||||
help="示向度舍入裕量（服务器保留两位小数；默认 0.005°）")
|
||||
run.add_argument("--real-budget-s", type=float, default=None,
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
"""分层强化学习实验：整场子目标调度环境与独立的八方向到达实验。
|
||||
|
||||
与 :mod:`mcm26b.rl`（单步 PPO 选点）的差异：高层在整场 10–16 源的统一
|
||||
候选槽位上做调度决策（证书清除 / 信息增益定位 / 覆盖站扫频 / 后备清除），
|
||||
当前运行闭环将子目标端点提交协议，八方向策略仅用于可选本地路径留痕，
|
||||
不影响真实请求及虚拟计时。完整高层训练、CLI 与配对评估仍待接通；
|
||||
优化架构及逐项验收见 docs/hrl-sol-execution.md。
|
||||
|
||||
信念（belief + 网格幸存集）、证书审批与后备清除复用 planner 基础设施；
|
||||
每次改动仍须验证真源保留和真实全清，不能用内部状态自洽替代评测。
|
||||
|
||||
本包属于训练/推理侧组件，运行时对依赖保持惰性（stable-baselines3 /
|
||||
sb3-contrib 仅在显式加载模型时导入）。
|
||||
"""
|
||||
@@ -0,0 +1,49 @@
|
||||
"""HRL 观测/动作空间常量：模型接口的唯一字面量来源。
|
||||
|
||||
槽位布局（训练与推理共享，改动任一值都会使已训练模型失配，须同步重训）：
|
||||
|
||||
0–19 clear 频道 c 的证书清除（槽位 = c-1，证书就绪才有效）
|
||||
20–39 locate 频道 c 的定位测量（信息增益选点或定向遮挡梯子探点）
|
||||
40–88 station 覆盖站（问题3：七点环 0–6 有效；问题4：49 点网格）
|
||||
89–108 fallback 频道 c 的后备网格清除（仅问题4，遮挡梯子耗尽后开放）
|
||||
109–119 保留（恒屏蔽）
|
||||
|
||||
高层观测 = 全局 8 维 + 20 频道 × 14 维 + 120 槽位 × 8 维 = 1248 维；
|
||||
低层观测 10 维、动作 Discrete(8)（8 方向，步长 ≤100 m）。
|
||||
"""
|
||||
|
||||
FEATURE_VERSION = 4
|
||||
ACTION_LAYOUT_VERSION = 1
|
||||
REWARD_VERSION = 1
|
||||
CHANNEL_COUNT = 20
|
||||
|
||||
# ------------------------------------------------------------ 高层动作槽位
|
||||
CLEAR_SLOTS = CHANNEL_COUNT
|
||||
LOCATE_SLOTS = CHANNEL_COUNT
|
||||
STATION_SLOTS = 49
|
||||
FALLBACK_SLOTS = CHANNEL_COUNT
|
||||
CLEAR_OFFSET = 0
|
||||
LOCATE_OFFSET = CLEAR_OFFSET + CLEAR_SLOTS
|
||||
STATION_OFFSET = LOCATE_OFFSET + LOCATE_SLOTS
|
||||
FALLBACK_OFFSET = STATION_OFFSET + STATION_SLOTS
|
||||
N_CANDIDATES = 120
|
||||
|
||||
# ------------------------------------------------------------ 高层观测
|
||||
PER_CHANNEL_DIM = 14
|
||||
HIGH_GLOBAL_DIM = 8
|
||||
PER_CANDIDATE_DIM = 8
|
||||
HIGH_OBS_DIM = (HIGH_GLOBAL_DIM + CHANNEL_COUNT * PER_CHANNEL_DIM
|
||||
+ N_CANDIDATES * PER_CANDIDATE_DIM) # 1248
|
||||
|
||||
# ------------------------------------------------------------ 低层接口
|
||||
LOW_OBS_DIM = 10
|
||||
LOW_NUM_ACTIONS = 8 # 8 方向，45° 等分
|
||||
LOW_STEP_M = 100.0 # 每步最大步长 [题面 低层动作 50–100 m]
|
||||
LOW_REACH_M = 20.0 # 到达阈值（与清除半径一致）
|
||||
LOW_MAX_STEPS = 60 # 直径穿越 ~51 步上界
|
||||
|
||||
# ------------------------------------------------------------ 观测归一化
|
||||
POS_SCALE_M = 2000.0 # 位置/位移归一化尺度（Ω 半径 1800 m 外扩）
|
||||
RADIUS_SCALE_M = 2000.0
|
||||
COST_SCALE_S = 600.0 # 槽位预计服务耗时归一化尺度
|
||||
MEASURE_COUNT_SCALE = 12.0
|
||||
@@ -0,0 +1,109 @@
|
||||
"""课程学习与域随机化：训练场景按阶段由易到难（prompt §7）。
|
||||
|
||||
四阶段课程：1 源/5 频道/无噪声 → 3 源/10 频道/±0.5° → 8 源/16 频道/±1°
|
||||
→ 10–16 源/20 频道/±1°（问题4 最后阶段按比例混入定向源）。域随机化
|
||||
覆盖源位置（Ω 圆域均匀）、频道（互异抽样）、有效接收半径、朝向与逐局
|
||||
误差种子 [prompt §7.2]；机器狗初始位置固定 (0,0)——协议语义如此，
|
||||
随机化它反而制造训练-部署差异（与 v1"训练部署特征错位"教训一致）。
|
||||
|
||||
真值只进入训练/评估场景生成，不进入任何观测；策略端仅见模拟器响应。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from random import Random
|
||||
|
||||
from ..config import ProblemConfig
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Stage:
|
||||
"""课程阶段参数：源数范围、频道上限、示向度噪声与定向源比例。"""
|
||||
|
||||
name: str
|
||||
sources_min: int
|
||||
sources_max: int
|
||||
channel_max: int # 频道 > channel_max 的频道保证无源
|
||||
bearing_error_half_deg: float
|
||||
directional_ratio: float # 问题4 最后阶段定向源比例；问题3 恒 0
|
||||
recommended_timesteps: int
|
||||
|
||||
|
||||
STAGES: tuple[Stage, ...] = (
|
||||
Stage("s1-single", 1, 1, 5, 0.0, 0.0, 60_000),
|
||||
Stage("s2-few", 3, 3, 10, 0.5, 0.0, 80_000),
|
||||
Stage("s3-many", 8, 8, 16, 1.0, 0.0, 100_000),
|
||||
Stage("s4-full", 10, 16, 20, 1.0, 0.4, 140_000),
|
||||
)
|
||||
|
||||
|
||||
def stage_with_noise(config: ProblemConfig, stage: Stage) -> ProblemConfig:
|
||||
"""把阶段噪声写入 ProblemConfig（belief/PF 的保守裕量随之同步）。"""
|
||||
field = dataclasses.replace(
|
||||
config.field, bearing_error_half_deg=stage.bearing_error_half_deg)
|
||||
return dataclasses.replace(config, field=field)
|
||||
|
||||
|
||||
# 快速模式只减少选点评分的样本，不改变证明使用的幸存网格。29.3m 格距的
|
||||
# Voronoi 裕量约20.72m，已经超过20m清除半径，会使粒子证书永远不可达。
|
||||
# 默认使用生产配置；快速模式必须显式指定并在训练元数据记录。
|
||||
TRAINING_SCORE_SUBSAMPLE = 240
|
||||
TRAINING_MC_REPRESENTATIVES = 5
|
||||
|
||||
|
||||
def training_config(config: ProblemConfig, stage: Stage, *,
|
||||
fidelity: str = "production") -> ProblemConfig:
|
||||
"""阶段噪声配置；fast只降低评分采样量，production与部署同参数。"""
|
||||
if fidelity not in ("production", "fast"):
|
||||
raise ValueError("fidelity 必须是 production 或 fast")
|
||||
noisy = stage_with_noise(config, stage)
|
||||
if fidelity == "production":
|
||||
return noisy
|
||||
bayes = dataclasses.replace(
|
||||
noisy.bayes,
|
||||
score_subsample=TRAINING_SCORE_SUBSAMPLE,
|
||||
mc_representatives=TRAINING_MC_REPRESENTATIVES)
|
||||
return dataclasses.replace(noisy, bayes=bayes)
|
||||
|
||||
|
||||
def make_scene(rng: Random, config: ProblemConfig, problem: int,
|
||||
stage: Stage, *, domain_randomization: bool = True
|
||||
) -> tuple[list, list[int]]:
|
||||
"""生成一局课程场景，返回 (referee.Source 列表, 已占用频道列表)。
|
||||
|
||||
源位置在 Ω 圆域均匀分布、频道从 1..stage.channel_max 互异抽样、
|
||||
有效接收半径 [1000,1500] 均匀 [题面]；问题4 按 stage.directional_ratio
|
||||
混入定向源，朝向均匀。domain_randomization=False 时只用 rng 的确定
|
||||
序列（仍随机但可整场复现），供评估固定场景对照。
|
||||
"""
|
||||
from referee import Source
|
||||
|
||||
field = config.field
|
||||
count = rng.randint(stage.sources_min, stage.sources_max)
|
||||
channels = rng.sample(range(field.channel_min, stage.channel_max + 1), count)
|
||||
sources = []
|
||||
for channel in channels:
|
||||
radius = field.omega_radius_m * math.sqrt(rng.random())
|
||||
angle = rng.uniform(0.0, 2.0 * math.pi)
|
||||
r_eff = rng.uniform(field.r_eff_min_m, field.r_eff_max_m)
|
||||
directional = (problem == 4
|
||||
and rng.random() < stage.directional_ratio)
|
||||
heading = rng.uniform(0.0, 360.0) if directional else 0.0
|
||||
if not domain_randomization:
|
||||
# 固定化退耦：半径/朝向取区间代表值，保留位置随机（评估用）。
|
||||
r_eff = field.r_eff_min_m
|
||||
sources.append(Source(channel=channel, x=radius * math.cos(angle),
|
||||
y=radius * math.sin(angle), r_eff_m=r_eff,
|
||||
directional=directional, heading_deg=heading))
|
||||
return sources, channels
|
||||
|
||||
|
||||
def training_spec(config: ProblemConfig, stage: Stage, *,
|
||||
max_real_duration_s: float = 3600.0):
|
||||
"""带阶段噪声的 referee Spec（训练侧使用；正式对接以 /enter 为准）。"""
|
||||
from ..harness import referee_spec
|
||||
return referee_spec(stage_with_noise(config, stage),
|
||||
max_real_duration_s=max_real_duration_s)
|
||||
@@ -0,0 +1,113 @@
|
||||
"""低层到达执行器与观测：8 方向步进（prompt §4.2），训练/部署同源。
|
||||
|
||||
低层只负责"怎么去"：给定当前位置与子目标，输出离散 8 方向步进序列。
|
||||
步长每步 ≤ ``LOW_STEP_M``，近目标时缩短步长；到达阈值 ``LOW_REACH_M``
|
||||
与清除半径一致。推演完全本地，不与模拟器交互、不产生虚拟耗时——协议
|
||||
语义下移动耗时 = 两次动作位置间的直线距离 ÷ 速度，仅由终点决定。
|
||||
|
||||
本模块仅供独立导航实验和轨迹诊断；当前 HTTP 执行路径仍直接服务高层
|
||||
目标，不能据此声称低层 PPO 改善正式任务耗时。policy=None 使用解析
|
||||
直线控制器；PPO 推演超时不会免费跳到目标。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
import numpy as np
|
||||
|
||||
from ..config import Vec2
|
||||
from .constants import (LOW_MAX_STEPS, LOW_NUM_ACTIONS, LOW_OBS_DIM,
|
||||
LOW_REACH_M, LOW_STEP_M, POS_SCALE_M)
|
||||
|
||||
__all__ = ["build_low_observation", "LowLevelExecutor", "action_bearing",
|
||||
"navigation_transition"]
|
||||
|
||||
|
||||
def action_bearing(action: int) -> float:
|
||||
"""离散方向 -> 弧度方位（0 = +x，逆时针 45° 等分）。"""
|
||||
return float(action) * 2.0 * math.pi / LOW_NUM_ACTIONS
|
||||
|
||||
|
||||
def navigation_transition(position: Vec2, goal: Vec2, action: int, *,
|
||||
step_m: float = LOW_STEP_M) -> tuple[Vec2, float]:
|
||||
"""训练与轨迹推演共享的运动学；返回位置和实际位移长度。
|
||||
|
||||
方向仍由策略决定，近目标时仅缩短步长，不把位置吸附到目标。
|
||||
这是无障碍平面导航实验；不引入观测不可见的绝对位置裁剪。
|
||||
"""
|
||||
if not 0 <= int(action) < LOW_NUM_ACTIONS:
|
||||
raise ValueError("低层动作必须是 0..7")
|
||||
if not math.isfinite(step_m) or step_m <= 0:
|
||||
raise ValueError("step_m 必须是有限正数")
|
||||
distance = min(float(step_m), math.dist(position, goal))
|
||||
bearing = action_bearing(int(action))
|
||||
next_position = (position[0] + distance * math.cos(bearing),
|
||||
position[1] + distance * math.sin(bearing))
|
||||
return next_position, math.dist(position, next_position)
|
||||
|
||||
|
||||
def build_low_observation(position: Vec2, goal: Vec2, prev_action: int | None,
|
||||
steps: int) -> np.ndarray:
|
||||
"""低层观测（shape=(LOW_OBS_DIM,)，训练/推理逐位一致）。"""
|
||||
dx, dy = goal[0] - position[0], goal[1] - position[1]
|
||||
distance = math.hypot(dx, dy)
|
||||
bearing = math.atan2(dy, dx)
|
||||
prev_sin = prev_cos = 0.0
|
||||
if prev_action is not None:
|
||||
prev_bearing = action_bearing(prev_action)
|
||||
prev_sin, prev_cos = math.sin(prev_bearing), math.cos(prev_bearing)
|
||||
return np.asarray([
|
||||
dx / POS_SCALE_M,
|
||||
dy / POS_SCALE_M,
|
||||
distance / POS_SCALE_M,
|
||||
math.sin(bearing),
|
||||
math.cos(bearing),
|
||||
steps / LOW_MAX_STEPS,
|
||||
prev_sin,
|
||||
prev_cos,
|
||||
min(distance / LOW_STEP_M, 3.0) / 3.0,
|
||||
1.0,
|
||||
], dtype=np.float32)
|
||||
|
||||
|
||||
class LowLevelExecutor:
|
||||
"""子目标到达推演：低层 PPO 或直线控制器（policy=None）。"""
|
||||
|
||||
def __init__(self, policy=None, *, step_m: float = LOW_STEP_M,
|
||||
reach_m: float = LOW_REACH_M, max_steps: int = LOW_MAX_STEPS):
|
||||
self.policy = policy
|
||||
self.step_m = float(step_m)
|
||||
self.reach_m = float(reach_m)
|
||||
self.max_steps = int(max_steps)
|
||||
self.last_roll_info: dict = {}
|
||||
|
||||
def roll(self, start: Vec2, goal: Vec2) -> list[Vec2]:
|
||||
"""返回真实推演途经点；last_roll_info 如实记录到达/截断。"""
|
||||
goal = (float(goal[0]), float(goal[1]))
|
||||
if self.policy is None:
|
||||
self.last_roll_info = {"reached": True, "truncated": False,
|
||||
"final_distance_m": 0.0,
|
||||
"path_length_m": math.dist(start, goal)}
|
||||
return [goal]
|
||||
position = (float(start[0]), float(start[1]))
|
||||
waypoints: list[Vec2] = []
|
||||
prev_action: int | None = None
|
||||
path_length = 0.0
|
||||
for steps in range(self.max_steps):
|
||||
if math.dist(position, goal) <= self.reach_m:
|
||||
break
|
||||
observation = build_low_observation(position, goal, prev_action,
|
||||
steps)
|
||||
action = int(self.policy.select(observation))
|
||||
position, moved = navigation_transition(position, goal, action,
|
||||
step_m=self.step_m)
|
||||
path_length += moved
|
||||
waypoints.append(position)
|
||||
prev_action = action
|
||||
distance = math.dist(position, goal)
|
||||
self.last_roll_info = {"reached": distance <= self.reach_m,
|
||||
"truncated": distance > self.reach_m,
|
||||
"final_distance_m": distance,
|
||||
"path_length_m": path_length}
|
||||
return waypoints
|
||||
@@ -0,0 +1,261 @@
|
||||
"""高层 PPO 训练环境：整场 10–16 源的发现-定位-清除调度（prompt §2/§5.2）。
|
||||
|
||||
与 v2 :class:`~mcm26b.rl.env.LocalizationEnv`（单源 ≤16 步子任务）的本质
|
||||
差异：本环境的每一步是一次完整的高层调度决策（候选槽位选择 + 任务执行），
|
||||
环境语义与运行闭环 :class:`~mcm26b.planner.hrl_loop.HRLLoop` 共用同一
|
||||
混入层（hrl.scheduler），训练-部署观测逐位同源。
|
||||
|
||||
奖励（prompt §5.2 的可折算实现，权重见 :class:`HRLReward`）：
|
||||
r = -Δ虚拟耗时/20 + 100·清除 + 5·新发现 + 0.5·Σ熵减 - 2·冗余检测
|
||||
终止：全部频道清除/证明缺席（+50）；截断仍有未决频道（-100）；异常（-100）。
|
||||
主口径是虚拟耗时——与题目目标函数（最小化平均定位清除时间/虚拟总时间）
|
||||
一致，清除/发现/熵减是塑形项（消融实验逐项验证其必要性）。
|
||||
|
||||
真值只在 reset 布景时使用，绝不进入观测；感知读数全部来自
|
||||
referee.LocalSimulator（与官方 HTTP 模拟器同协议语义）。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from random import Random
|
||||
|
||||
import gymnasium as gym
|
||||
import numpy as np
|
||||
from gymnasium import spaces
|
||||
|
||||
from ..belief import ChannelStatus
|
||||
from ..config import ProblemConfig
|
||||
from ..hrl.curriculum import STAGES, Stage, make_scene, training_config
|
||||
from ..hrl.constants import HIGH_OBS_DIM, N_CANDIDATES
|
||||
from ..planner.hrl_loop import HRLLoop
|
||||
from ..protocol.client import SimulatorClient
|
||||
from ..protocol.log import SessionLog
|
||||
|
||||
__all__ = ["HRLReward", "HierarchicalEnv"]
|
||||
|
||||
|
||||
@dataclass
|
||||
class HRLReward:
|
||||
"""高层奖励权重。
|
||||
|
||||
默认保留 prompt §5.2 的可解释塑形，便于对照实验；P0.4 建议的
|
||||
纯时间口径可通过 :meth:`time_only` 显式启用，训练脚本默认使用它。
|
||||
"""
|
||||
|
||||
time_scale: float = 20.0 # r -= Δ虚拟耗时 / time_scale
|
||||
clear: float = 100.0 # 每清除一个源
|
||||
detect: float = 5.0 # 每新发现一个频道有信号
|
||||
info_weight: float = 0.5 # 粒子云熵减（log 幸存格数）内在奖励
|
||||
redundant: float = 2.0 # 同频道同点重复检测惩罚
|
||||
invalid: float = 2.0 # 掩码外动作（无掩码消融路径）
|
||||
terminal_accounted: float = 50.0 # 全部频道清除/证毕的收尾奖励
|
||||
terminal_unresolved: float = 100.0 # 截断/停机时仍有未决频道的惩罚
|
||||
failure: float = 100.0 # 协议/守恒性异常惩罚
|
||||
|
||||
@classmethod
|
||||
def time_only(cls) -> "HRLReward":
|
||||
"""P0.4 纯时间目标：只惩罚已发生虚拟时间，去掉塑形项。
|
||||
|
||||
终止奖/罚是固定常数，不逐源叠加；失败罚按虚拟时限/时间尺度
|
||||
设定为足以覆盖一整局时间成本。
|
||||
"""
|
||||
return cls(
|
||||
time_scale=100.0,
|
||||
clear=0.0,
|
||||
detect=0.0,
|
||||
info_weight=0.0,
|
||||
redundant=0.0,
|
||||
invalid=2.0,
|
||||
terminal_accounted=30.0,
|
||||
terminal_unresolved=300.0,
|
||||
failure=3600.0,
|
||||
)
|
||||
|
||||
|
||||
class HierarchicalEnv(gym.Env):
|
||||
"""整场分层调度 Gymnasium 环境（问题3/4，课程参数化）。
|
||||
|
||||
stage=None 时使用 STAGES 末阶段（完整任务口径）；课程训练按阶段
|
||||
传入。domain_randomization=False 保留随机位置但固定半径代表值，
|
||||
供配对评估。
|
||||
"""
|
||||
|
||||
metadata = {"render_modes": []}
|
||||
|
||||
def __init__(self, config: ProblemConfig | None = None, *,
|
||||
problem: int = 3, stage: Stage | None = None,
|
||||
max_episode_steps: int = 300,
|
||||
domain_randomization: bool = True,
|
||||
reward: HRLReward | None = None,
|
||||
real_budget_s: float = 3600.0,
|
||||
error_seed: int | None = None,
|
||||
fidelity: str = "production"):
|
||||
super().__init__()
|
||||
if problem not in (3, 4):
|
||||
raise ValueError("problem 必须是 3 或 4")
|
||||
self.base_config = config or ProblemConfig()
|
||||
self.problem = int(problem)
|
||||
self.stage = stage or STAGES[-1]
|
||||
self.max_episode_steps = int(max_episode_steps)
|
||||
self.domain_randomization = bool(domain_randomization)
|
||||
self.reward = reward or HRLReward()
|
||||
self.real_budget_s = float(real_budget_s)
|
||||
self.error_seed = error_seed
|
||||
self.fidelity = fidelity
|
||||
# 默认与部署同精度；fast仅减少评分采样，保留证书网格精度。
|
||||
self.train_config = training_config(self.base_config, self.stage,
|
||||
fidelity=fidelity)
|
||||
self.observation_space = spaces.Box(low=-12.0, high=12.0,
|
||||
shape=(HIGH_OBS_DIM,),
|
||||
dtype=np.float32)
|
||||
self.action_space = spaces.Discrete(N_CANDIDATES)
|
||||
|
||||
self._scene_rng = Random()
|
||||
self._loop: HRLLoop | None = None
|
||||
self._client: SimulatorClient | None = None
|
||||
self._steps = 0
|
||||
self._terminated = False
|
||||
self._truncated = False
|
||||
self._start_virtual_s = 0.0
|
||||
self._current_candidates: list = []
|
||||
self._failure_reason: str | None = None
|
||||
self._last_outcome = None
|
||||
|
||||
# ------------------------------------------------------------ 场景
|
||||
def _make_episode(self) -> None:
|
||||
from ..harness import LocalSimTransport
|
||||
from referee import LocalSimulator
|
||||
|
||||
sources, _channels = make_scene(
|
||||
self._scene_rng, self.train_config, self.problem, self.stage,
|
||||
domain_randomization=self.domain_randomization)
|
||||
self._simulator = LocalSimulator(
|
||||
sources, robot_id="hrl-train",
|
||||
spec=self._training_spec(),
|
||||
error_seed=(self.error_seed if self.error_seed is not None
|
||||
else self._scene_rng.randint(0, 2 ** 31 - 1)),
|
||||
stress_mode=True)
|
||||
self._client = SimulatorClient(
|
||||
"hrl-train", config=self.train_config,
|
||||
transport=LocalSimTransport(self._simulator), log=SessionLog())
|
||||
self._client.enter()
|
||||
self._loop = HRLLoop(self._client, self.train_config,
|
||||
problem=self.problem)
|
||||
|
||||
def _training_spec(self):
|
||||
from ..harness import referee_spec
|
||||
from ..hrl.curriculum import training_spec
|
||||
return training_spec(self.train_config, self.stage,
|
||||
max_real_duration_s=self.real_budget_s)
|
||||
|
||||
# ------------------------------------------------------------ Gym API
|
||||
def reset(self, *, seed: int | None = None, options: dict | None = None):
|
||||
super().reset(seed=seed)
|
||||
if seed is not None:
|
||||
self._scene_rng = Random(seed)
|
||||
self._steps = 0
|
||||
self._terminated = False
|
||||
self._truncated = False
|
||||
self._failure_reason = None
|
||||
self._make_episode()
|
||||
self._start_virtual_s = self._client.state.virtual_time_s
|
||||
self._current_candidates = self._loop.build_candidates()
|
||||
observation = self._loop.hrl_observation(self._current_candidates)
|
||||
return observation, self._info()
|
||||
|
||||
def action_masks(self) -> np.ndarray:
|
||||
"""MaskablePPO 动作掩码接口（与观测同源的候选槽位有效性）。"""
|
||||
return self._loop.hrl_action_mask(self._current_candidates)
|
||||
|
||||
def _info(self) -> dict:
|
||||
loop = self._loop
|
||||
belief = loop.belief
|
||||
cleared = len(loop.cleared)
|
||||
absent = sum(1 for ch in belief.channels.values()
|
||||
if ch.status is ChannelStatus.ABSENT)
|
||||
return {
|
||||
"steps": self._steps,
|
||||
"cleared": cleared,
|
||||
"absent_proven": absent,
|
||||
"accounted": cleared + absent,
|
||||
"virtual_time_s": self._client.state.virtual_time_s,
|
||||
"episode_virtual_cost_s": (self._client.state.virtual_time_s
|
||||
- self._start_virtual_s),
|
||||
"stop_reason": loop.stop_reason,
|
||||
"failure_reason": self._failure_reason,
|
||||
"anomalies": list(loop.anomalies),
|
||||
}
|
||||
|
||||
def step(self, action):
|
||||
if self._terminated or self._truncated:
|
||||
raise RuntimeError("step() 需要在 terminated/truncated 后先 reset()")
|
||||
loop = self._loop
|
||||
self._steps += 1
|
||||
try:
|
||||
outcome = loop.hrl_step(action, candidates=self._current_candidates)
|
||||
except Exception as exc: # 协议/守恒性异常：与 v2 环境同口径收尾
|
||||
self._failure_reason = type(exc).__name__ + ': ' + str(exc)
|
||||
observation = np.zeros((HIGH_OBS_DIM,), dtype=np.float32)
|
||||
self._terminated = True
|
||||
return (observation, -self.reward.failure, True, False,
|
||||
self._info())
|
||||
self._last_outcome = outcome
|
||||
|
||||
if outcome.invalid:
|
||||
reward = -self.reward.invalid
|
||||
else:
|
||||
reward = self._reward_from(outcome)
|
||||
|
||||
# 终止判定：停机原因/全部证毕；或候选耗尽时走运行闭环 run() 的
|
||||
# 同款安全网（覆盖缺席标记 + 阻塞定位清除），安全网的虚拟耗时
|
||||
# 计入本步奖励（归因给促成该局面的决策序列）。
|
||||
terminated = loop.stop_reason is not None or loop._all_accounted()
|
||||
if not terminated:
|
||||
self._current_candidates = loop.build_candidates()
|
||||
if not self._current_candidates:
|
||||
virtual_before = self._client.state.virtual_time_s
|
||||
loop._mark_absent_after_coverage()
|
||||
loop._resolve_detected()
|
||||
reward -= (self._client.state.virtual_time_s - virtual_before) \
|
||||
/ self.reward.time_scale
|
||||
self._current_candidates = []
|
||||
terminated = True
|
||||
else:
|
||||
self._current_candidates = []
|
||||
|
||||
if terminated:
|
||||
reward += (self.reward.terminal_accounted
|
||||
if loop._all_accounted()
|
||||
else -self.reward.terminal_unresolved)
|
||||
truncated = False
|
||||
if not terminated and self._steps >= self.max_episode_steps:
|
||||
truncated = True
|
||||
if not loop._all_accounted():
|
||||
reward -= self.reward.terminal_unresolved
|
||||
self._terminated = terminated
|
||||
self._truncated = truncated
|
||||
if terminated or truncated:
|
||||
observation = np.zeros((HIGH_OBS_DIM,), dtype=np.float32)
|
||||
else:
|
||||
self._current_candidates = loop.build_candidates()
|
||||
observation = loop.hrl_observation(self._current_candidates)
|
||||
return observation, float(reward), terminated, truncated, self._info()
|
||||
|
||||
def _reward_from(self, outcome) -> float:
|
||||
reward = -outcome.delta_virtual_s / self.reward.time_scale
|
||||
reward += self.reward.clear * len(outcome.clear_success)
|
||||
reward += self.reward.detect * len(outcome.new_detections)
|
||||
reward += self.reward.info_weight * sum(outcome.info_gains.values())
|
||||
if outcome.redundant:
|
||||
reward -= self.reward.redundant
|
||||
return reward
|
||||
|
||||
def close(self):
|
||||
client = getattr(self, "_client", None)
|
||||
if client is not None and client.state.entered \
|
||||
and not client.state.exited:
|
||||
try:
|
||||
client.exit()
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,158 @@
|
||||
"""低层 PPO 训练环境：随机起点-子目标的 8 方向到达任务（prompt §4.2/§5.1）。
|
||||
|
||||
奖励为实际位移耗时惩罚、转向惩罚与到达奖励。步长上限 100m，近目标
|
||||
缩短步长，避免固定步长振荡。此环境仅用于独立导航实验；正式 HTTP
|
||||
路径尚不执行低层 PPO 的途经点，不能将导航成绩当成正式任务收益。
|
||||
|
||||
策略端只见相对位姿观测，真值（子目标坐标）经相对向量进入观测，
|
||||
与部署执行器完全同源（build_low_observation）。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from random import Random
|
||||
|
||||
import gymnasium as gym
|
||||
import numpy as np
|
||||
from gymnasium import spaces
|
||||
|
||||
from ..config import ProblemConfig
|
||||
from .constants import (LOW_MAX_STEPS, LOW_NUM_ACTIONS, LOW_OBS_DIM,
|
||||
LOW_REACH_M, LOW_STEP_M, POS_SCALE_M)
|
||||
from .executor import (action_bearing, build_low_observation,
|
||||
navigation_transition)
|
||||
|
||||
__all__ = ["LowLevelEnv"]
|
||||
|
||||
|
||||
class LowLevelEnv(gym.Env):
|
||||
"""低层到达任务：观测 10 维，动作 Discrete(8)，逐步计费。"""
|
||||
|
||||
metadata = {"render_modes": []}
|
||||
|
||||
def __init__(self, config: ProblemConfig | None = None, *,
|
||||
seed: int | None = None,
|
||||
step_m: float = LOW_STEP_M, reach_m: float = LOW_REACH_M,
|
||||
max_steps: int = LOW_MAX_STEPS,
|
||||
min_distance_m: float = 300.0,
|
||||
turn_penalty: float = 0.05, reach_bonus: float = 2.0,
|
||||
step_cost_scale: float = 100.0,
|
||||
truncation_penalty: float = 2.0):
|
||||
super().__init__()
|
||||
self.config = config or ProblemConfig()
|
||||
self.step_m = float(step_m)
|
||||
self.reach_m = float(reach_m)
|
||||
self.max_steps = int(max_steps)
|
||||
self.min_distance_m = float(min_distance_m)
|
||||
self.turn_penalty = float(turn_penalty)
|
||||
self.reach_bonus = float(reach_bonus)
|
||||
self.step_cost_scale = float(step_cost_scale)
|
||||
self.truncation_penalty = float(truncation_penalty)
|
||||
self.observation_space = spaces.Box(low=-np.inf, high=np.inf,
|
||||
shape=(LOW_OBS_DIM,),
|
||||
dtype=np.float32)
|
||||
self.action_space = spaces.Discrete(LOW_NUM_ACTIONS)
|
||||
self._rng = Random(seed)
|
||||
self._position = (0.0, 0.0)
|
||||
self._goal = (0.0, 0.0)
|
||||
self._steps = 0
|
||||
self._prev_action: int | None = None
|
||||
self._terminated = False
|
||||
self._truncated = False
|
||||
self._path_length_m = 0.0
|
||||
self._virtual_time_s = 0.0
|
||||
self._straight_m = 0.0
|
||||
|
||||
# ------------------------------------------------------------ Gym API
|
||||
def reset(self, *, seed: int | None = None, options: dict | None = None):
|
||||
super().reset(seed=seed)
|
||||
if seed is not None:
|
||||
self._rng = Random(seed)
|
||||
radius = self.config.field.omega_radius_m
|
||||
self._position = self._sample_point(radius)
|
||||
for _ in range(100):
|
||||
self._goal = self._sample_point(radius)
|
||||
if math.dist(self._position, self._goal) >= self.min_distance_m:
|
||||
break
|
||||
self._steps = 0
|
||||
self._prev_action = None
|
||||
self._terminated = False
|
||||
self._truncated = False
|
||||
self._path_length_m = 0.0
|
||||
self._virtual_time_s = 0.0
|
||||
self._straight_m = math.dist(self._position, self._goal)
|
||||
return self._observe(), self._info()
|
||||
|
||||
def action_masks(self) -> np.ndarray:
|
||||
"""无障碍低层八个方向均可用，兼容 MaskablePPO。"""
|
||||
return np.ones(LOW_NUM_ACTIONS, dtype=bool)
|
||||
|
||||
def _sample_point(self, radius: float) -> tuple[float, float]:
|
||||
angle = self._rng.uniform(0.0, 2.0 * math.pi)
|
||||
distance = radius * math.sqrt(self._rng.random())
|
||||
return (distance * math.cos(angle), distance * math.sin(angle))
|
||||
|
||||
def step(self, action):
|
||||
if self._terminated or self._truncated:
|
||||
raise RuntimeError("step() 需要在 terminated/truncated 后先 reset()")
|
||||
action = int(action)
|
||||
if self._prev_action is not None:
|
||||
turn = abs((action_bearing(action)
|
||||
- action_bearing(self._prev_action) + math.pi)
|
||||
% (2.0 * math.pi) - math.pi) / math.pi
|
||||
else:
|
||||
turn = 0.0
|
||||
self._position, moved = navigation_transition(
|
||||
self._position, self._goal, action, step_m=self.step_m)
|
||||
self._prev_action = action
|
||||
self._steps += 1
|
||||
|
||||
step_cost_s = moved / self.config.cost.speed_mps
|
||||
self._path_length_m += moved
|
||||
self._virtual_time_s += step_cost_s
|
||||
reward = -step_cost_s / self.step_cost_scale
|
||||
reward -= self.turn_penalty * turn
|
||||
distance = math.dist(self._position, self._goal)
|
||||
terminated = distance <= self.reach_m
|
||||
truncated = False
|
||||
if terminated:
|
||||
reward += self.reach_bonus
|
||||
elif self._steps >= self.max_steps:
|
||||
truncated = True
|
||||
reward -= self.truncation_penalty
|
||||
self._terminated = terminated
|
||||
self._truncated = truncated
|
||||
info = self._info()
|
||||
info["final_distance_m"] = distance
|
||||
info["step_distance_m"] = moved
|
||||
info["step_virtual_time_s"] = step_cost_s
|
||||
return self._observe(), float(reward), terminated, truncated, info
|
||||
|
||||
def _observe(self) -> np.ndarray:
|
||||
return build_low_observation(self._position, self._goal,
|
||||
self._prev_action, self._steps)
|
||||
|
||||
def _info(self) -> dict:
|
||||
return {
|
||||
"position": self._position,
|
||||
"goal": self._goal,
|
||||
"steps": self._steps,
|
||||
"distance_m": math.dist(self._position, self._goal),
|
||||
"straight_m": self._straight_m,
|
||||
"path_length_m": self._path_length_m,
|
||||
"virtual_time_s": self._virtual_time_s,
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------ 专家（BC）
|
||||
def expert_action(self) -> int:
|
||||
"""直线专家：选择方位最接近子目标方向的离散方向（BC 标签）。"""
|
||||
bearing = math.atan2(self._goal[1] - self._position[1],
|
||||
self._goal[0] - self._position[0])
|
||||
best, best_gap = 0, float("inf")
|
||||
for action in range(LOW_NUM_ACTIONS):
|
||||
gap = abs((action_bearing(action) - bearing + math.pi)
|
||||
% (2.0 * math.pi) - math.pi)
|
||||
if gap < best_gap:
|
||||
best, best_gap = action, gap
|
||||
return best
|
||||
@@ -0,0 +1,135 @@
|
||||
"""HRL 策略加载与推理（运行时接入 planner 的唯一入口，模式同 rl.policy）。
|
||||
|
||||
模型文件由 scripts/train_hrl_low.py / train_hrl_high.py 生成，并伴随同名
|
||||
JSON 元数据（feature_version/algorithm/problem/维度）；加载时校验，失配
|
||||
即拒绝运行，杜绝静默错位。stable-baselines3 / torch 均为惰性导入。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
from .constants import (ACTION_LAYOUT_VERSION, FEATURE_VERSION,
|
||||
HIGH_OBS_DIM, LOW_NUM_ACTIONS, LOW_OBS_DIM,
|
||||
N_CANDIDATES, REWARD_VERSION)
|
||||
|
||||
__all__ = ["HighLevelPolicy", "LowLevelPolicy",
|
||||
"default_high_model_path", "default_low_model_path"]
|
||||
|
||||
|
||||
def _resolve_device(device: str) -> str:
|
||||
if device != "auto":
|
||||
return device
|
||||
try:
|
||||
import torch
|
||||
return "cuda" if torch.cuda.is_available() else "cpu"
|
||||
except ImportError:
|
||||
return "cpu"
|
||||
|
||||
|
||||
def _load_maskable(model_path: Path, device: str, *,
|
||||
obs_dim: int, act_dim: int, problem: int):
|
||||
if not model_path.is_file():
|
||||
raise RuntimeError(
|
||||
f"未找到 HRL 模型 {model_path}。请先训练：\n"
|
||||
f" uv sync --group rl --group dev\n"
|
||||
f" uv run python scripts/train_hrl_high.py --problem {problem}")
|
||||
try:
|
||||
from sb3_contrib import MaskablePPO
|
||||
except ImportError as exc:
|
||||
raise RuntimeError(
|
||||
"缺少 RL 运行依赖（sb3-contrib）。请执行 `uv sync --group rl` "
|
||||
"后重试。") from exc
|
||||
meta_path = model_path.with_suffix('.json')
|
||||
metadata = json.loads(meta_path.read_text(encoding='utf-8')) \
|
||||
if meta_path.exists() else {}
|
||||
# 高层模型必须携带完整语义元数据；缺字段不能默认兼容，防止旧槽位/
|
||||
# 旧奖励语义模型被静默加载。低层导航模型只校验基础特征版本。
|
||||
strict = (act_dim == N_CANDIDATES)
|
||||
if strict:
|
||||
required = ("feature_version", "action_layout_version",
|
||||
"reward_version", "algorithm", "problem",
|
||||
"obs_dim", "act_dim")
|
||||
missing = [key for key in required if key not in metadata]
|
||||
if missing:
|
||||
raise RuntimeError(
|
||||
f"HRL 高层模型元数据缺少字段 {missing}；请用当前 "
|
||||
f"scripts/train_hrl_high.py 重训。")
|
||||
if metadata.get("feature_version") != FEATURE_VERSION:
|
||||
raise RuntimeError(
|
||||
f"HRL 模型特征版本失配: 期望 {FEATURE_VERSION}，"
|
||||
f"实际 {metadata.get('feature_version')}。请用当前版本重训。")
|
||||
if metadata.get("action_layout_version") != ACTION_LAYOUT_VERSION:
|
||||
raise RuntimeError(
|
||||
f"HRL 模型动作布局版本失配: 期望 {ACTION_LAYOUT_VERSION}，"
|
||||
f"实际 {metadata.get('action_layout_version')}。请重训。")
|
||||
if metadata.get("reward_version") != REWARD_VERSION:
|
||||
raise RuntimeError(
|
||||
f"HRL 模型奖励版本失配: 期望 {REWARD_VERSION}，"
|
||||
f"实际 {metadata.get('reward_version')}。请重训。")
|
||||
else:
|
||||
if metadata.get("feature_version") != FEATURE_VERSION:
|
||||
raise RuntimeError(
|
||||
f"低层模型特征版本失配: 期望 {FEATURE_VERSION}，"
|
||||
f"实际 {metadata.get('feature_version')}。请重训。")
|
||||
if metadata.get("algorithm") != "maskable_ppo":
|
||||
raise RuntimeError("HRL 模型必须是 maskable_ppo 训练产物")
|
||||
if problem is not None and metadata.get("problem") not in (None, problem):
|
||||
raise RuntimeError("模型训练问题编号与当前第三/四问不匹配")
|
||||
model = MaskablePPO.load(str(model_path), device=device)
|
||||
try:
|
||||
loaded_obs = model.observation_space.shape[0]
|
||||
loaded_act = model.action_space.n
|
||||
except Exception as exc:
|
||||
raise RuntimeError(f"模型空间与 HRL 接口不兼容: {model_path}") from exc
|
||||
if loaded_obs != obs_dim or loaded_act != act_dim:
|
||||
raise RuntimeError(
|
||||
f"模型接口失配: 期望 obs={obs_dim}/act={act_dim}，"
|
||||
f"实际 obs={loaded_obs}/act={loaded_act}。请重训。")
|
||||
return model
|
||||
|
||||
|
||||
class HighLevelPolicy:
|
||||
"""高层子目标调度策略：观测+掩码 -> 槽位编号（确定性 argmax）。"""
|
||||
|
||||
def __init__(self, model_path: str | Path, *, device: str = "auto",
|
||||
problem: int | None = None):
|
||||
self.device = _resolve_device(device)
|
||||
self.path = Path(model_path)
|
||||
self.model = _load_maskable(self.path, self.device, obs_dim=HIGH_OBS_DIM,
|
||||
act_dim=N_CANDIDATES, problem=problem)
|
||||
meta_path = self.path.with_suffix('.json')
|
||||
self.metadata = json.loads(meta_path.read_text(encoding='utf-8')) \
|
||||
if meta_path.exists() else {}
|
||||
|
||||
def select(self, observation: np.ndarray, action_masks: np.ndarray) -> int:
|
||||
action, _state = self.model.predict(
|
||||
observation, action_masks=action_masks, deterministic=True)
|
||||
return int(action)
|
||||
|
||||
|
||||
class LowLevelPolicy:
|
||||
"""低层到达策略：相对位姿观测 -> 8 方向（确定性 argmax）。"""
|
||||
|
||||
def __init__(self, model_path: str | Path, *, device: str = "auto"):
|
||||
self.device = _resolve_device(device)
|
||||
self.path = Path(model_path)
|
||||
self.model = _load_maskable(self.path, self.device, obs_dim=LOW_OBS_DIM,
|
||||
act_dim=LOW_NUM_ACTIONS, problem=None)
|
||||
|
||||
def select(self, observation: np.ndarray) -> int:
|
||||
action, _state = self.model.predict(observation, deterministic=True)
|
||||
return int(action)
|
||||
|
||||
|
||||
def default_high_model_path(problem: int) -> Path:
|
||||
if problem not in (3, 4):
|
||||
raise ValueError("problem 必须是 3 或 4")
|
||||
return Path("output/hrl") / f"q{problem}_hrl_high.zip"
|
||||
|
||||
|
||||
def default_low_model_path() -> Path:
|
||||
return Path("output/hrl") / "hrl_low.zip"
|
||||
@@ -0,0 +1,493 @@
|
||||
"""高层调度核心：候选槽位构建、观测、动作掩码与任务执行。
|
||||
|
||||
本模块是 :class:`~mcm26b.planner.hrl_loop.HRLLoop`（运行闭环）与
|
||||
:class:`~mcm26b.hrl.hierarchical_env.HierarchicalEnv`（训练环境）共用的
|
||||
混入层，保证训练-部署特征逐位同源（v1"训练部署特征错位"审核教训）。
|
||||
|
||||
宿主须为 :class:`~mcm26b.planner.bayes_loop.BayesLoop` 派生闭环，提供
|
||||
client/belief/particles/ledger/_record_measure 等感知与执行原语；本层只做
|
||||
三件事——
|
||||
|
||||
1. 把整场调度决策离散化为固定槽位（constants.N_CANDIDATES）：
|
||||
clear(20) + locate(20) + station(49) + fallback(20) + 保留(11)；
|
||||
2. 由可见观测（belief + 网格幸存集 + 覆盖账本）构造观测与动作掩码，
|
||||
绝不读取源真值；
|
||||
3. 执行选中候选并报告事实（虚拟耗时增量、新发现、熵减、清除结果），
|
||||
奖励折算由训练环境负责。
|
||||
|
||||
Q4 定向源语义：no_signal 不能删除空间候选（可能是朝向遮挡），故问题4 的
|
||||
幸存集只按 direction/near 更新；遮挡证据走 blocked_probes 梯子与后备清除
|
||||
槽位，与 DirectionalExtension 的保守口径一致。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass, field
|
||||
from typing import NamedTuple
|
||||
|
||||
import numpy as np
|
||||
|
||||
from ..belief import ChannelStatus
|
||||
from ..config import ProblemConfig, Vec2
|
||||
from ..protocol.rules import CLEAR_SUCCESS, DIRECTION, NEAR, NO_SIGNAL
|
||||
from ..protocol.timing import move_time
|
||||
from .constants import (CHANNEL_COUNT, CLEAR_OFFSET, COST_SCALE_S,
|
||||
FALLBACK_OFFSET, HIGH_GLOBAL_DIM, HIGH_OBS_DIM,
|
||||
LOCATE_OFFSET, MEASURE_COUNT_SCALE, N_CANDIDATES,
|
||||
PER_CANDIDATE_DIM, PER_CHANNEL_DIM, POS_SCALE_M,
|
||||
RADIUS_SCALE_M, STATION_OFFSET)
|
||||
from ..planner import fallback_clear_cells, safe_disk_radius
|
||||
|
||||
__all__ = ["Candidate", "StepOutcome", "HRLControllerMixin", "point_key"]
|
||||
|
||||
|
||||
def point_key(point: Vec2) -> tuple[int, int]:
|
||||
"""候选点去重键：0.5m 网格取整（与 bayes_loop 同口径）。"""
|
||||
return (int(round(point[0] * 2)), int(round(point[1] * 2)))
|
||||
|
||||
|
||||
class Candidate(NamedTuple):
|
||||
"""一个高层候选子目标。slot 为固定槽位编号，供动作掩码与特征定位。"""
|
||||
|
||||
slot: int
|
||||
kind: str # "clear" | "locate" | "probe" | "station" | "fallback"
|
||||
position: Vec2
|
||||
channel: int | None = None
|
||||
station_index: int | None = None
|
||||
service_s: float = 0.0 # 预计服务耗时（含换频，不含移动）
|
||||
targets: int = 1 # station: 待测频道数；fallback: 格数；其余 1
|
||||
|
||||
|
||||
@dataclass
|
||||
class StepOutcome:
|
||||
"""一次高层决策-执行的客观结果（奖励折算由训练环境负责）。"""
|
||||
|
||||
action: int
|
||||
kind: str | None = None
|
||||
channel: int | None = None
|
||||
invalid: bool = False # 掩码外动作（无掩码消融路径）：未与模拟器交互
|
||||
delta_virtual_s: float = 0.0
|
||||
new_detections: tuple[int, ...] = ()
|
||||
info_gains: dict[int, float] = field(default_factory=dict) # 频道 -> 熵减(log)
|
||||
clear_success: tuple[int, ...] = ()
|
||||
redundant: bool = False
|
||||
position_reached: Vec2 | None = None
|
||||
|
||||
|
||||
class HRLControllerMixin:
|
||||
"""高层调度混入：依赖宿主（HRLLoop）提供 BayesLoop 感知与执行原语。
|
||||
|
||||
宿主须在 ``__init__`` 初始化：``_obs_mec_cache={}``、
|
||||
``_station_pending_map={}``、``_initial_alive={channel: len(pf)}``
|
||||
（观测侧缓存与归一化基准；证书相关计算仍用 pf.mec() 精确值）。
|
||||
"""
|
||||
|
||||
problem: int
|
||||
|
||||
# ------------------------------------------------------------ 候选构建
|
||||
def build_candidates(self) -> list[Candidate]:
|
||||
"""构建当前全部有效候选（固定槽位；训练/推理/heuristic 专家同源）。
|
||||
|
||||
副作用：缓存每频道证书（``self._cert_cache``）与站点待测集
|
||||
（``self._station_pending_map``）供观测构建复用；调用方须在本
|
||||
函数之后、任何状态变更之前构建观测。
|
||||
"""
|
||||
config = self.config
|
||||
detected = self.belief.channels_by_status(ChannelStatus.DETECTED)
|
||||
by_slot: dict[int, Candidate] = {}
|
||||
cert_cache: dict[int, object] = {}
|
||||
station_map: dict[int, list[int]] = {}
|
||||
|
||||
# --- clear 槽位：证书就绪且未因失败等待补测
|
||||
for channel in detected:
|
||||
cert = self._best_certificate(channel)
|
||||
cert_cache[channel] = cert
|
||||
if cert is not None and cert.certified \
|
||||
and channel not in self._clear_failed:
|
||||
by_slot[CLEAR_OFFSET + channel - 1] = Candidate(
|
||||
CLEAR_OFFSET + channel - 1, "clear", tuple(cert.center),
|
||||
channel=channel,
|
||||
service_s=config.cost.clear_success_s, targets=1)
|
||||
|
||||
# --- locate / probe 槽位：未就绪清除的已发现频道
|
||||
for channel in detected:
|
||||
if CLEAR_OFFSET + channel - 1 in by_slot:
|
||||
continue
|
||||
region = self.belief.candidate_set(channel)
|
||||
if region is None:
|
||||
continue
|
||||
blocked = self.blocked_probes.get(channel, []) \
|
||||
if self.problem == 4 else []
|
||||
if blocked:
|
||||
point = self._hrl_probe_point(region, channel)
|
||||
kind = "probe"
|
||||
else:
|
||||
point = self._select_point(channel)
|
||||
kind = "locate"
|
||||
if point is None:
|
||||
continue
|
||||
by_slot[LOCATE_OFFSET + channel - 1] = Candidate(
|
||||
LOCATE_OFFSET + channel - 1, kind, tuple(point), channel=channel,
|
||||
service_s=config.cost.measure_s + config.cost.switch_s, targets=1)
|
||||
|
||||
# --- station 槽位：尚有 UNKNOWN 待测频道的覆盖站
|
||||
for index, point in enumerate(self.ledger.points):
|
||||
pending = self._station_pending(index)
|
||||
station_map[index] = pending
|
||||
if not pending:
|
||||
continue
|
||||
switches = len(pending) - (1 if self.client.state.channel in pending else 0)
|
||||
by_slot[STATION_OFFSET + index] = Candidate(
|
||||
STATION_OFFSET + index, "station", tuple(point),
|
||||
station_index=index,
|
||||
service_s=(config.cost.measure_s * len(pending)
|
||||
+ config.cost.switch_s * max(switches, 0)),
|
||||
targets=len(pending))
|
||||
|
||||
# --- fallback 槽位（仅问题4）：遮挡梯子耗尽且格数在预算内
|
||||
if self.problem == 4:
|
||||
for channel in detected:
|
||||
blocked = self.blocked_probes.get(channel, [])
|
||||
if len(blocked) < config.search.max_blocked_probes:
|
||||
continue
|
||||
region = self.belief.candidate_set(channel)
|
||||
if region is None:
|
||||
continue
|
||||
cells = fallback_clear_cells(region.polygon, config)
|
||||
if not cells or len(cells) > config.search.fallback_max_cells:
|
||||
continue
|
||||
by_slot[FALLBACK_OFFSET + channel - 1] = Candidate(
|
||||
FALLBACK_OFFSET + channel - 1, "fallback",
|
||||
tuple(region.mec()[0]), channel=channel,
|
||||
service_s=config.cost.clear_fail_s * len(cells),
|
||||
targets=len(cells))
|
||||
|
||||
self._cert_cache = cert_cache
|
||||
self._station_pending_map = station_map
|
||||
self._last_candidates = list(by_slot.values())
|
||||
return self._last_candidates
|
||||
|
||||
def candidate_by_slot(self, candidates: list[Candidate],
|
||||
action: int) -> Candidate | None:
|
||||
for cand in candidates:
|
||||
if cand.slot == int(action):
|
||||
return cand
|
||||
return None
|
||||
|
||||
# ------------------------------------------------------------ 动作掩码
|
||||
def hrl_action_mask(self, candidates: list[Candidate]) -> np.ndarray:
|
||||
"""槽位有效性掩码：无候选槽位恒屏蔽（训练/推理共用）。"""
|
||||
mask = np.zeros(N_CANDIDATES, dtype=bool)
|
||||
for cand in candidates:
|
||||
mask[cand.slot] = True
|
||||
return mask
|
||||
|
||||
# ------------------------------------------------------------ 观测
|
||||
def _hrl_obs_mec(self, channel: int) -> tuple[Vec2, float]:
|
||||
"""观测用 MEC：幸存子采样上的最小包围圆，按 pf.version 缓存。
|
||||
|
||||
仅作策略观测特征（近似 MEU/σ），证书相关计算一律用 pf.mec()
|
||||
精确值——两者分开缓存，互不影响。
|
||||
"""
|
||||
pf = self.particles[channel]
|
||||
cached = self._obs_mec_cache.get(channel)
|
||||
if cached is not None and cached[0] == pf.version:
|
||||
return cached[1], cached[2]
|
||||
points = pf.subsample(600)
|
||||
if not points:
|
||||
return (0.0, 0.0), 0.0
|
||||
from ..geometry.localization import smallest_enclosing_circle
|
||||
center, radius = smallest_enclosing_circle(points)
|
||||
self._obs_mec_cache[channel] = (pf.version, center, radius)
|
||||
return center, radius
|
||||
|
||||
def _hrl_channel_status_index(self, channel: int) -> int:
|
||||
"""六状态编码：UNKNOWN/DETECTED/TRACKING/LOCATED/CLEARED/ABSENT。"""
|
||||
status = self.belief.channels[channel].status
|
||||
if status is ChannelStatus.UNKNOWN:
|
||||
return 0
|
||||
if status is ChannelStatus.CLEARED:
|
||||
return 4
|
||||
if status is ChannelStatus.ABSENT:
|
||||
return 5
|
||||
cert = getattr(self, "_cert_cache", {}).get(channel)
|
||||
if cert is not None and getattr(cert, "certified", False):
|
||||
return 3
|
||||
_, radius = self._hrl_obs_mec(channel)
|
||||
return 1 if radius > 500.0 else 2
|
||||
|
||||
def hrl_observation(self, candidates: list[Candidate]) -> np.ndarray:
|
||||
"""构造高层观测（shape=(HIGH_OBS_DIM,)，训练/推理逐位一致）。"""
|
||||
config = self.config
|
||||
field = config.field
|
||||
position = self.client.state.position
|
||||
belief = self.belief
|
||||
|
||||
detected_count = len(belief.channels_by_status(ChannelStatus.DETECTED))
|
||||
accounted = sum(1 for ch in belief.channels.values()
|
||||
if ch.status in (ChannelStatus.CLEARED,
|
||||
ChannelStatus.ABSENT))
|
||||
cleared = len(self.cleared)
|
||||
glob = [
|
||||
position[0] / POS_SCALE_M,
|
||||
position[1] / POS_SCALE_M,
|
||||
min(self.client.state.virtual_time_s, field.max_virtual_duration_s)
|
||||
/ field.max_virtual_duration_s,
|
||||
cleared / 16.0,
|
||||
accounted / CHANNEL_COUNT,
|
||||
(self.client.state.channel - field.channel_min)
|
||||
/ max(field.channel_max - field.channel_min, 1),
|
||||
float(self.problem == 4),
|
||||
detected_count / 16.0,
|
||||
]
|
||||
assert len(glob) == HIGH_GLOBAL_DIM
|
||||
|
||||
pending_station = np.zeros(CHANNEL_COUNT + 1, dtype=bool)
|
||||
for index, channels in getattr(self, "_station_pending_map", {}).items():
|
||||
for channel in channels:
|
||||
pending_station[channel] = True
|
||||
|
||||
channel_features: list[float] = []
|
||||
for channel in range(field.channel_min, field.channel_max + 1):
|
||||
ch = belief.channels[channel]
|
||||
status6 = [0.0] * 6
|
||||
status6[self._hrl_channel_status_index(channel)] = 1.0
|
||||
pf = self.particles[channel]
|
||||
mec_visible = ch.status is ChannelStatus.DETECTED
|
||||
if mec_visible:
|
||||
center, radius = self._hrl_obs_mec(channel)
|
||||
else:
|
||||
center, radius = (0.0, 0.0), 0.0
|
||||
initial = max(self._initial_alive.get(channel, 1), 1)
|
||||
channel_features.extend([
|
||||
*status6,
|
||||
center[0] / POS_SCALE_M,
|
||||
center[1] / POS_SCALE_M,
|
||||
radius / RADIUS_SCALE_M,
|
||||
math.log(len(pf) + 1) / math.log(initial + 1)
|
||||
if initial > 1 else 0.0,
|
||||
min(len(ch.measurements), MEASURE_COUNT_SCALE * 2)
|
||||
/ MEASURE_COUNT_SCALE / 2,
|
||||
math.hypot(center[0] - position[0], center[1] - position[1])
|
||||
/ POS_SCALE_M if mec_visible else 0.0,
|
||||
len(self.blocked_probes.get(channel, []))
|
||||
/ max(config.search.max_blocked_probes, 1)
|
||||
if self.problem == 4 else 0.0,
|
||||
1.0 if (ch.status is ChannelStatus.UNKNOWN
|
||||
and pending_station[channel]) else 0.0,
|
||||
])
|
||||
assert len(channel_features) == CHANNEL_COUNT * PER_CHANNEL_DIM
|
||||
|
||||
by_slot = {cand.slot: cand for cand in candidates}
|
||||
candidate_features: list[float] = []
|
||||
for slot in range(N_CANDIDATES):
|
||||
cand = by_slot.get(slot)
|
||||
if cand is None:
|
||||
candidate_features.extend([0.0] * PER_CANDIDATE_DIM)
|
||||
continue
|
||||
dx, dy = cand.position[0] - position[0], cand.position[1] - position[1]
|
||||
move_s = move_time(position, cand.position, config.cost)
|
||||
candidate_features.extend([
|
||||
1.0 if cand.kind == "clear" else 0.0,
|
||||
1.0 if cand.kind in ("locate", "probe") else 0.0,
|
||||
1.0 if cand.kind == "station" else 0.0,
|
||||
1.0 if cand.kind == "fallback" else 0.0,
|
||||
dx / POS_SCALE_M,
|
||||
dy / POS_SCALE_M,
|
||||
min((move_s + cand.service_s) / COST_SCALE_S, 2.0),
|
||||
min(cand.targets / 16.0, 1.0),
|
||||
])
|
||||
assert len(candidate_features) == N_CANDIDATES * PER_CANDIDATE_DIM
|
||||
|
||||
observation = np.asarray(glob + channel_features + candidate_features,
|
||||
dtype=np.float32)
|
||||
assert observation.shape == (HIGH_OBS_DIM,)
|
||||
return observation
|
||||
|
||||
# ------------------------------------------------------------ 启发式专家
|
||||
def heuristic_action(self, candidates: list[Candidate]) -> int:
|
||||
"""把 BayesLoop 任务排序映射为槽位（BC 专家 + 无策略降级路径）。
|
||||
|
||||
任务表与排序完全复用 :meth:`BayesLoop._build_tasks` /
|
||||
:meth:`BayesLoop._order_tasks`；映射失败（如 Q4 梯子任务不在
|
||||
任务表中）退化为最近有效候选。
|
||||
"""
|
||||
by_slot = {cand.slot: cand for cand in candidates}
|
||||
if not by_slot:
|
||||
raise RuntimeError("heuristic_action：当前无可用候选")
|
||||
try:
|
||||
order = self._order_tasks(self._build_tasks())
|
||||
first = order[0]
|
||||
if first.kind == "clear" and first.channel is not None:
|
||||
slot = CLEAR_OFFSET + first.channel - 1
|
||||
elif first.kind == "locate" and first.channel is not None:
|
||||
slot = LOCATE_OFFSET + first.channel - 1
|
||||
else:
|
||||
slot = STATION_OFFSET + (first.station_index or 0)
|
||||
if slot in by_slot:
|
||||
return slot
|
||||
except Exception:
|
||||
pass
|
||||
position = self.client.state.position
|
||||
return min(by_slot, key=lambda s: (
|
||||
math.dist(position, by_slot[s].position), s))
|
||||
|
||||
# ------------------------------------------------------------ 决策执行
|
||||
def hrl_step(self, action: int,
|
||||
candidates: list[Candidate] | None = None) -> StepOutcome:
|
||||
"""执行一次高层决策；candidates 为空则现场构建（训练环境逐位同源）。"""
|
||||
action = int(action)
|
||||
self._reset_step_recording()
|
||||
old_virtual = self.client.state.virtual_time_s
|
||||
if candidates is None:
|
||||
candidates = self.build_candidates()
|
||||
cand = self.candidate_by_slot(candidates, action)
|
||||
if cand is None:
|
||||
return StepOutcome(action=action, invalid=True)
|
||||
if self.stop_reason is not None:
|
||||
return StepOutcome(action=action, kind=cand.kind,
|
||||
channel=cand.channel, invalid=True)
|
||||
|
||||
if cand.kind == "station":
|
||||
self._execute_station(cand.station_index)
|
||||
elif cand.kind in ("locate", "probe"):
|
||||
self._hrl_execute_locate(cand)
|
||||
elif cand.kind == "clear":
|
||||
self._hrl_execute_clear(cand)
|
||||
elif cand.kind == "fallback":
|
||||
self._hrl_execute_fallback(cand)
|
||||
else: # pragma: no cover - 布局守卫
|
||||
raise ValueError(f"未知候选类型: {cand.kind}")
|
||||
|
||||
outcome = StepOutcome(action=action, kind=cand.kind,
|
||||
channel=cand.channel,
|
||||
position_reached=tuple(self.client.state.position))
|
||||
outcome.delta_virtual_s = (self.client.state.virtual_time_s - old_virtual)
|
||||
new_detections: list[int] = []
|
||||
info_gains: dict[int, float] = {}
|
||||
redundant = False
|
||||
for event in self._step_events:
|
||||
if event["was_unknown"] and event["result"] in (DIRECTION, NEAR):
|
||||
new_detections.append(event["channel"])
|
||||
gain = math.log10(event["pf_before"] + 1) - math.log10(event["pf_after"] + 1)
|
||||
info_gains[event["channel"]] = (info_gains.get(event["channel"], 0.0)
|
||||
+ min(max(gain, 0.0), 2.0))
|
||||
redundant = redundant or event["redundant"]
|
||||
outcome.new_detections = tuple(new_detections)
|
||||
outcome.info_gains = info_gains
|
||||
outcome.redundant = redundant
|
||||
outcome.clear_success = tuple(self._step_clears)
|
||||
self._step_clears = []
|
||||
return outcome
|
||||
|
||||
# ------------------------------------------------------------ 执行原语
|
||||
def _hrl_execute_locate(self, cand: Candidate) -> None:
|
||||
"""定位测量（含 Q4 梯子探点）：记录证据，不自动清除（高层决策）。"""
|
||||
config = self.config
|
||||
channel = cand.channel
|
||||
pf = self.particles[channel]
|
||||
before = len(pf)
|
||||
outcome = self.client.measure(cand.position, channel)
|
||||
self._record_measure(channel, cand.position, outcome)
|
||||
self._may_clear.add(channel)
|
||||
if self.problem == 4:
|
||||
if outcome.result == NO_SIGNAL:
|
||||
if channel not in self.directional_observed:
|
||||
region = self.belief.candidate_set(channel)
|
||||
safe = safe_disk_radius(region, config) if region else -1.0
|
||||
if safe >= 0 and math.dist(cand.position, region.mec()[0]) <= safe:
|
||||
self.directional_observed.append(channel)
|
||||
self.blocked_probes.setdefault(channel, []).append(cand.position)
|
||||
else:
|
||||
self.blocked_probes.pop(channel, None)
|
||||
if len(pf) == before:
|
||||
self._stall[channel] = self._stall.get(channel, 0) + 1
|
||||
self.no_shrink_measures += 1
|
||||
else:
|
||||
self._stall[channel] = 0
|
||||
|
||||
def _hrl_execute_clear(self, cand: Candidate) -> None:
|
||||
"""证书清除：证书失效（不应被掩码允许）时静默跳过。"""
|
||||
cert = self._best_certificate(cand.channel)
|
||||
if cert is None or not cert.certified:
|
||||
self._anomaly("hrl_clear_without_certificate", cand.channel)
|
||||
return
|
||||
if self._attempt_clear(cand.channel, cert.center, cert):
|
||||
self._clear_failed.discard(cand.channel)
|
||||
self._stall.pop(cand.channel, None)
|
||||
else:
|
||||
self._clear_failed.add(cand.channel)
|
||||
self._may_clear.discard(cand.channel)
|
||||
|
||||
def _hrl_execute_fallback(self, cand: Candidate) -> None:
|
||||
"""Q4 后备网格清除（与 DirectionalExtension._fallback_clear 同口径）。"""
|
||||
channel = cand.channel
|
||||
region = self.belief.candidate_set(channel)
|
||||
if region is None:
|
||||
return
|
||||
config, session = self.config, self.config.session
|
||||
cells = fallback_clear_cells(region.polygon, config)
|
||||
entry = {"channel": channel, "cells": len(cells)}
|
||||
self.fallback_log.append(entry)
|
||||
if len(cells) > config.search.fallback_max_cells:
|
||||
entry["outcome"] = "skipped_cell_cap"
|
||||
return
|
||||
remaining = self.client.real_remaining_s
|
||||
if remaining is not None and len(cells) * session.real_rtt_p95_s \
|
||||
+ session.reserve_exit_s > remaining:
|
||||
entry["outcome"] = "skipped_budget"
|
||||
return
|
||||
remaining_cells = list(cells)
|
||||
attempts = 0
|
||||
while remaining_cells:
|
||||
if self._virtual_cap_reached():
|
||||
entry["outcome"] = "skipped_virtual_budget"
|
||||
entry["attempts"] = attempts
|
||||
return
|
||||
cell = min(remaining_cells, key=lambda c: (
|
||||
math.dist(self.client.state.position, c), c))
|
||||
remaining_cells.remove(cell)
|
||||
attempts += 1
|
||||
outcome = self.client.clear(cell, channel)
|
||||
if outcome.result == CLEAR_SUCCESS:
|
||||
self.belief.mark_cleared(channel)
|
||||
self.cleared.append(channel)
|
||||
self._step_clears.append(channel)
|
||||
self.blocked_probes.pop(channel, None)
|
||||
entry["outcome"] = "cleared"
|
||||
entry["attempts"] = attempts
|
||||
return
|
||||
entry["outcome"] = "exhausted"
|
||||
entry["attempts"] = attempts
|
||||
self._anomaly("hrl_fallback_exhausted", channel, cells=len(cells))
|
||||
|
||||
def _hrl_probe_point(self, region, channel: int) -> Vec2:
|
||||
"""定向遮挡梯子探点（与 DirectionalExtension._next_probe 同口径）。"""
|
||||
from ..planner import select_measurement_point
|
||||
blocked = self.blocked_probes.get(channel, [])
|
||||
center, radius = region.mec()
|
||||
if not blocked:
|
||||
return select_measurement_point(region, self.client.state.position,
|
||||
self.config).position
|
||||
safe = safe_disk_radius(region, self.config)
|
||||
first = blocked[0]
|
||||
if len(blocked) == 1 and safe >= 0:
|
||||
dx, dy = center[0] - first[0], center[1] - first[1]
|
||||
norm = math.hypot(dx, dy)
|
||||
if norm <= 1e-9:
|
||||
return (center[0] + safe, center[1])
|
||||
scale = min(1.0, safe / norm)
|
||||
return (center[0] + dx * scale, center[1] + dy * scale)
|
||||
base = math.atan2(first[1] - center[1], first[0] - center[0])
|
||||
azimuth = base + math.radians(45.0 * (len(blocked) - 1))
|
||||
return (center[0] + radius * math.cos(azimuth),
|
||||
center[1] + radius * math.sin(azimuth))
|
||||
|
||||
# ------------------------------------------------------------ 事件捕获
|
||||
def _reset_step_recording(self) -> None:
|
||||
self._step_events = []
|
||||
self._step_clears = []
|
||||
|
||||
def _record_event(self, **event) -> None:
|
||||
self._step_events.append(event)
|
||||
@@ -0,0 +1,174 @@
|
||||
"""P7 分层强化学习闭环（``--strategy hrl`` / ``--q4-strategy hrl``）。
|
||||
|
||||
边界与保守性声明（与 rl_loop.py 同口径）：
|
||||
- 高层只在固定候选槽位上做调度决策（证书清除 / 信息增益定位 / 覆盖站 /
|
||||
Q4 梯子探点与后备清除），候选几何全部由既有规则生成；
|
||||
- 感知（belief + 网格幸存集）、清除证书、跳测规则、异常留痕完全继承
|
||||
:class:`BayesLoop`，不触碰真源保留定理的前提；
|
||||
- 问题3 的幸存集按全部读数更新（no_signal 删盘）；问题4 定向源的
|
||||
no_signal 不能排除朝向遮挡，幸存集只按 direction/near 更新，遮挡
|
||||
证据走 blocked_probes 梯子与后备清除槽位（DirectionalExtension 口径）；
|
||||
- 高层策略缺失时经 :meth:`heuristic_action` 降级为 BayesLoop 任务排序，
|
||||
闭环仍然完备可运行。
|
||||
|
||||
问题4 发现覆盖使用 49 点网格（七点环的覆盖证明只对全向源成立）。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from ..belief import ChannelStatus
|
||||
from ..config import ProblemConfig, Vec2
|
||||
from ..coverage import CoverageLedger, coverage_points
|
||||
from ..hrl.scheduler import HRLControllerMixin, point_key
|
||||
from ..protocol.rules import NO_SIGNAL
|
||||
from .bayes_loop import BayesLoop
|
||||
|
||||
|
||||
class HRLLoop(HRLControllerMixin, BayesLoop):
|
||||
"""分层闭环：BayesLoop 感知/证书 + 高层槽位调度（问题3/4 共用）。"""
|
||||
|
||||
def __init__(self, client, config: ProblemConfig = ProblemConfig(), *,
|
||||
problem: int = 3, high_policy=None, low_policy=None,
|
||||
trace_executor: bool = False):
|
||||
super().__init__(client, config)
|
||||
if problem not in (3, 4):
|
||||
raise ValueError("problem 必须是 3 或 4")
|
||||
self.problem = int(problem)
|
||||
self.high_policy = high_policy
|
||||
self.low_policy = low_policy
|
||||
self.trace_executor = bool(trace_executor)
|
||||
self.executor_trace: list[dict] = []
|
||||
self.hrl_decisions: list[dict] = []
|
||||
self._step_events: list[dict] = []
|
||||
self._step_clears: list[int] = []
|
||||
self._cert_cache: dict = {}
|
||||
self._last_candidates: list = []
|
||||
self._obs_mec_cache: dict = {}
|
||||
self._station_pending_map: dict = {}
|
||||
# 观测里幸存格数的归一化基准（初始全集），使特征对训练/部署的
|
||||
# 网格密度差不变。
|
||||
self._initial_alive = {channel: len(pf)
|
||||
for channel, pf in self.particles.items()}
|
||||
if problem == 4:
|
||||
self.directional_observed: list[int] = []
|
||||
self.blocked_probes: dict[int, list[Vec2]] = {}
|
||||
self.fallback_log: list[dict] = []
|
||||
# 49 点网格：七点环覆盖证明只对全向源成立 [口径见模块注释]
|
||||
self.ledger = CoverageLedger(config, points=coverage_points(config))
|
||||
|
||||
# ------------------------------------------------------------ 主循环
|
||||
def _coverage_phase(self) -> None:
|
||||
"""统一任务循环：每步构建候选，由高层策略（或启发式降级）选槽执行。"""
|
||||
while self.stop_reason is None:
|
||||
if self._count_limit_reached() or self._virtual_cap_reached():
|
||||
return
|
||||
# 顺路观测会改变信念、证书和候选；必须发生在本轮快照之前。
|
||||
if self.high_policy is None and self.problem == 3 \
|
||||
and self.config.bayes.opportunistic_measure:
|
||||
self._measure_at_stop()
|
||||
if self.stop_reason is not None or self._count_limit_reached() \
|
||||
or self._virtual_cap_reached():
|
||||
return
|
||||
candidates = self.build_candidates()
|
||||
if not candidates:
|
||||
return
|
||||
if self.high_policy is not None:
|
||||
observation = self.hrl_observation(candidates)
|
||||
mask = self.hrl_action_mask(candidates)
|
||||
action = self.high_policy.select(observation, mask)
|
||||
else:
|
||||
# 降级路径：BayesLoop 任务排序（无 RL 依赖时闭环仍完备）
|
||||
action = self.heuristic_action(candidates)
|
||||
if self.candidate_by_slot(candidates, action) is None:
|
||||
# 非法策略输出不能造成零耗时无限循环；事实记入异常再安全调度。
|
||||
self._anomaly("hrl_invalid_policy_action", 0, action=int(action))
|
||||
action = self.heuristic_action(candidates)
|
||||
cand = self.candidate_by_slot(candidates, action)
|
||||
if cand is not None:
|
||||
self._log_low_level(cand)
|
||||
self.hrl_decisions.append({
|
||||
"action": int(action), "kind": cand.kind,
|
||||
"channel": cand.channel,
|
||||
"virtual_time_s": self.client.state.virtual_time_s})
|
||||
self.hrl_step(action, candidates=candidates)
|
||||
|
||||
# ------------------------------------------------------------ 感知记录
|
||||
def _record_measure(self, channel: int, position: Vec2, outcome) -> None:
|
||||
"""BayesLoop._record_measure 的事件捕获版；问题4 no_signal 不删格。
|
||||
|
||||
Q4 语义：no_signal 可能是朝向遮挡 [观测推理]，删除幸存格会破坏
|
||||
真源保留，因此问题4 幸存集只按 direction/near 更新；缺席证明
|
||||
交由 49 点网格覆盖账本（继承的 _mark_absent_after_coverage）。
|
||||
"""
|
||||
key = point_key(position)
|
||||
redundant = key in self._measured_points.get(channel, set())
|
||||
belief = self.belief.channels[channel]
|
||||
was_unknown = belief.status is ChannelStatus.UNKNOWN
|
||||
pf = self.particles[channel]
|
||||
before = len(pf)
|
||||
self.belief.update_measure(channel, position, outcome.result,
|
||||
outcome.svd_deg)
|
||||
self._measured_points.setdefault(channel, set()).add(key)
|
||||
self._clear_failed.discard(channel)
|
||||
belief = self.belief.channels[channel]
|
||||
if belief.status not in (ChannelStatus.CLEARED, ChannelStatus.ABSENT):
|
||||
if not (self.problem == 4 and outcome.result == NO_SIGNAL):
|
||||
pf.apply_measure(position, outcome.result, outcome.svd_deg)
|
||||
if not len(pf):
|
||||
if belief.status is ChannelStatus.UNKNOWN:
|
||||
self.belief.mark_absent(channel)
|
||||
if channel not in self.pf_absent_channels:
|
||||
self.pf_absent_channels.append(channel)
|
||||
else:
|
||||
self._anomaly("pf_empty_while_detected", channel,
|
||||
position=position)
|
||||
self._record_event(channel=channel, result=outcome.result,
|
||||
pf_before=before, pf_after=len(pf),
|
||||
redundant=redundant, was_unknown=was_unknown)
|
||||
|
||||
def _attempt_clear(self, channel: int, target: Vec2, cert) -> bool:
|
||||
success = super()._attempt_clear(channel, target, cert)
|
||||
if success and hasattr(self, "blocked_probes"):
|
||||
self.blocked_probes.pop(channel, None)
|
||||
if success:
|
||||
self._step_clears.append(channel)
|
||||
return success
|
||||
|
||||
# ------------------------------------------------------------ 低层执行留痕
|
||||
def _log_low_level(self, cand) -> None:
|
||||
"""低层到达推演留痕（本地步进，不与模拟器交互，不产生虚拟耗时）。"""
|
||||
if not self.trace_executor:
|
||||
return
|
||||
from ..hrl.executor import LowLevelExecutor
|
||||
executor = getattr(self, "_executor", None)
|
||||
if executor is None:
|
||||
executor = self._executor = LowLevelExecutor(self.low_policy)
|
||||
waypoints = executor.roll(self.client.state.position, cand.position)
|
||||
if self.low_policy is not None:
|
||||
self.executor_trace.append({
|
||||
"kind": cand.kind, "channel": cand.channel,
|
||||
"goal": [round(v, 2) for v in cand.position],
|
||||
"waypoints": [[round(v, 2) for v in p] for p in waypoints],
|
||||
"navigation": dict(executor.last_roll_info),
|
||||
})
|
||||
|
||||
# ------------------------------------------------------------ 收尾
|
||||
def _summary(self, info) -> dict:
|
||||
report = super()._summary(info)
|
||||
report["strategy"] = "hrl_q4" if self.problem == 4 else "hrl"
|
||||
report["hrl_problem"] = self.problem
|
||||
kinds: dict[str, int] = {}
|
||||
for decision in self.hrl_decisions:
|
||||
kinds[decision["kind"]] = kinds.get(decision["kind"], 0) + 1
|
||||
report["hrl_action_kinds"] = kinds
|
||||
report["hrl_decisions"] = self.hrl_decisions
|
||||
report["hrl_executor_trace"] = self.executor_trace
|
||||
if self.problem == 4:
|
||||
report["directional_observed_channels"] = \
|
||||
list(self.directional_observed)
|
||||
report["fallback_clears"] = self.fallback_log
|
||||
report["fallback_clear_attempts"] = sum(
|
||||
entry.get("attempts", 0) for entry in self.fallback_log)
|
||||
return report
|
||||
@@ -0,0 +1,15 @@
|
||||
"""模型训练前端：本地回环 Web UI + 训练子进程监控。
|
||||
|
||||
启动：
|
||||
python -m mcm26b.training_ui --ui-port 8036
|
||||
或：
|
||||
python scripts/train_hrl_ui.py --ui-port 8036
|
||||
|
||||
子命令：
|
||||
python -m mcm26b.training_ui --print-schema # 配置契约
|
||||
python -m mcm26b.training_ui --print-preview # 离线示例快照
|
||||
"""
|
||||
from .server import TrainingUI, TrainingJob, run_training_ui
|
||||
from .schema import config_schema
|
||||
|
||||
__all__ = ["TrainingUI", "TrainingJob", "run_training_ui", "config_schema"]
|
||||
@@ -0,0 +1,4 @@
|
||||
from .server import run_training_ui
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(run_training_ui())
|
||||
@@ -0,0 +1,211 @@
|
||||
"""训练前端示例快照：把一段"看起来像真的" PPO 训练曲线合成为合法快照。
|
||||
|
||||
用途有两个，都不涉及真实训练：
|
||||
1. ``GET /api/train/state?preview=1`` 让界面在没有训练进程时也能验证渲染
|
||||
（曲线、指标卡、进程时间线、日志着色、逐回合表）；
|
||||
2. 单元测试可以断言前端的收敛判读分支（上升/平坦/KL 过猛）各有稳定输入。
|
||||
|
||||
曲线是解析函数 + 固定种子的确定性噪声，不含任何真实评测数据。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import random
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
__all__ = ["DEMO_CONFIG", "DEMO_SCENARIOS", "demo_metrics", "demo_monitor",
|
||||
"demo_log_lines", "demo_snapshot"]
|
||||
|
||||
DEMO_SCENARIOS = ("rising", "plateau", "kl-spike")
|
||||
|
||||
DEMO_CONFIG: dict[str, Any] = {
|
||||
"problem": 3, "stage": 2, "timesteps": 12000, "seed": 320001,
|
||||
"envs": 4, "vec_env": "subproc", "device": "mps",
|
||||
"config_profile": "production", "reward_profile": "time",
|
||||
"model": "output/hrl/ui/demo/q3.zip", "init": None, "resume": False,
|
||||
"eval_episodes": 2, "rollout_steps": 256, "batch_size": 256,
|
||||
"n_epochs": 10, "max_wall_s": 1200.0, "local_selector": "geometry",
|
||||
"local_model": None, "verbose": 1,
|
||||
}
|
||||
|
||||
DEMO_COMMAND = [
|
||||
"python", "scripts/train_hrl_high.py", "--problem", "3", "--stage", "2",
|
||||
"--timesteps", "12000", "--seed", "320001", "--envs", "4",
|
||||
"--vec-env", "subproc", "--device", "mps", "--config-profile", "production",
|
||||
"--reward-profile", "time", "--model", "output/hrl/ui/demo/q3.zip",
|
||||
"--eval-episodes", "2", "--rollout-steps", "256", "--batch-size", "256",
|
||||
"--n-epochs", "10", "--max-wall-s", "1200.0", "--local-selector",
|
||||
"geometry", "--progress-json", "--verbose", "1",
|
||||
]
|
||||
|
||||
DEMO_STEPS = 48
|
||||
|
||||
|
||||
def _finite(value: float) -> float | None:
|
||||
if value != value or value in (float("inf"), float("-inf")):
|
||||
return None
|
||||
return round(value, 8)
|
||||
|
||||
|
||||
def demo_metrics(count: int = DEMO_STEPS, steps: int = 12000, *,
|
||||
seed: int = 7, scenario: str = "rising",
|
||||
total_steps: int | None = None, fps: float = 420.0
|
||||
) -> list[dict[str, Any]]:
|
||||
"""合成 count 个进度点，横轴覆盖到 ``total_steps`` 的 ``count/DEMO_STEPS``。
|
||||
|
||||
场景（用于验证收敛判读的各个分支，不含真实数据）：
|
||||
rising 奖励对数饱和上升、KL 正常 → "收敛趋势良好"
|
||||
plateau 奖励已进入平台期 → "已趋于平稳（疑似收敛）"
|
||||
kl-spike 末段近似 KL 冲高 → "KL 偏大，策略更新可能过猛"
|
||||
"""
|
||||
if scenario not in DEMO_SCENARIOS:
|
||||
raise ValueError(f"未知示例场景: {scenario}")
|
||||
total = int(total_steps if total_steps is not None else steps)
|
||||
rng = random.Random(seed)
|
||||
points: list[dict[str, Any]] = []
|
||||
per = total / DEMO_STEPS
|
||||
ceiling = 42.0
|
||||
span = min(1.0, max(1e-6, count / float(DEMO_STEPS)))
|
||||
for index in range(1, count + 1):
|
||||
t = (index / float(DEMO_STEPS)) / span # 折到整段训练的进度比例
|
||||
steps_now = int(round(index * per))
|
||||
if scenario == "plateau":
|
||||
reward = 41.5 + rng.uniform(-0.35, 0.35)
|
||||
value_loss_curve = 3.6 + rng.uniform(-0.2, 0.2)
|
||||
approx_kl = max(0.001, 0.004 + rng.uniform(-0.001, 0.001))
|
||||
explained = min(0.95, 0.88 + rng.uniform(-0.02, 0.02))
|
||||
elif scenario == "kl-spike":
|
||||
reward = ceiling * (1.0 - math.exp(-3.1 * t)) + rng.uniform(-1.2, 1.2)
|
||||
value_loss_curve = 62.0 * math.exp(-2.4 * t) + 3.4
|
||||
base_kl = 0.031 * math.exp(-1.7 * t)
|
||||
approx_kl = base_kl + (0.075 if t > 0.8 else 0.0) + rng.uniform(-0.003, 0.003)
|
||||
explained = -0.09 + 1.05 * (1.0 - math.exp(-2.2 * t))
|
||||
else:
|
||||
noise = rng.uniform(-1.4, 1.4) * (1.0 - 0.6 * t)
|
||||
reward = ceiling * (1.0 - math.exp(-3.1 * t)) + noise
|
||||
value_loss_curve = 62.0 * math.exp(-2.4 * t) + 3.4
|
||||
approx_kl = 0.031 * math.exp(-1.7 * t)
|
||||
explained = -0.09 + 1.05 * (1.0 - math.exp(-2.2 * t))
|
||||
value_loss = value_loss_curve + rng.uniform(-0.5, 0.5)
|
||||
policy_loss = -0.021 + 0.006 * math.sin(index / 4.0) + rng.uniform(-0.002, 0.002)
|
||||
entropy = 1.35 - 0.55 * (t if scenario != "plateau" else 0.9) \
|
||||
+ rng.uniform(-0.02, 0.02)
|
||||
points.append({
|
||||
"timesteps": steps_now,
|
||||
"n_updates": index * 12,
|
||||
"ep_rew_mean": _finite(reward),
|
||||
"ep_len_mean": _finite(24.0 + 21.0 * t + rng.uniform(-1.5, 1.5)),
|
||||
"value_loss": _finite(value_loss),
|
||||
"policy_gradient_loss": _finite(policy_loss),
|
||||
"approx_kl": _finite(max(0.0008, approx_kl + rng.uniform(-0.002, 0.002))),
|
||||
"entropy_loss": _finite(entropy),
|
||||
"explained_variance": _finite(min(0.97, explained
|
||||
+ rng.uniform(-0.03, 0.03))),
|
||||
# 吞吐尽量接近真实量级（envs=4 时约 400+ 步/秒），否则剩余时间
|
||||
# 估计会给出荒谬的 ETA
|
||||
"fps": _finite(max(30.0, fps * (0.92 + 0.16 * t)
|
||||
+ rng.uniform(-12.0, 12.0))),
|
||||
})
|
||||
return points
|
||||
|
||||
|
||||
def demo_monitor(count: int = 60, seed: int = 11) -> list[dict[str, Any]]:
|
||||
rng = random.Random(seed)
|
||||
rows = []
|
||||
clock = 0.0
|
||||
for index in range(count):
|
||||
t = index / max(1, count - 1)
|
||||
clock += rng.uniform(4.0, 14.0)
|
||||
rows.append({
|
||||
"r": _finite(38.0 * (1.0 - math.exp(-3.0 * t)) + rng.uniform(-3.0, 3.0)),
|
||||
"l": float(rng.randint(28, 74)),
|
||||
"t": round(clock, 4),
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def demo_log_lines() -> list[str]:
|
||||
"""一段包含各类着色分支的日志：SB3 表格、进度 JSON、警告与收尾。"""
|
||||
lines = [
|
||||
"=== 训练前端离线预览：以下为示例快照，非真实训练输出 ===",
|
||||
"使用设备: mps",
|
||||
"创建 4 个 DummyVecEnv/SubprocVecEnv 环境 (stage=2, problem=3)",
|
||||
"Monitor CSV: output/hrl/ui/demo/high_curve_q3_env0.monitor.csv",
|
||||
"-----------------------------",
|
||||
"| rollout/ | |",
|
||||
"| ep_len_mean | 31.4 |",
|
||||
"| ep_rew_mean | 12.83 |",
|
||||
"| time/ | |",
|
||||
"| fps | 412 |",
|
||||
"| total_timesteps | 512 |",
|
||||
"-----------------------------",
|
||||
]
|
||||
for index in range(1, 9):
|
||||
t = index / 8.0
|
||||
lines.append(
|
||||
'{"event": "progress", "timesteps": %d, "n_updates": %d, '
|
||||
'"ep_rew_mean": %.3f, "ep_len_mean": %.2f, "value_loss": %.4f, '
|
||||
'"policy_gradient_loss": %.5f, "approx_kl": %.5f, '
|
||||
'"entropy_loss": %.4f, "explained_variance": %.4f, "fps": %.1f}'
|
||||
% (index * 1500, index * 12, 42.0 * (1 - math.exp(-3.1 * t)),
|
||||
24 + 21 * t, 62.0 * math.exp(-2.4 * t) + 3.4, -0.021 + 0.006 * t,
|
||||
0.031 * math.exp(-1.7 * t), 1.35 - 0.55 * t,
|
||||
-0.09 + 1.05 * (1 - math.exp(-2.2 * t)), 390 + 70 * t)
|
||||
)
|
||||
lines += [
|
||||
"UserWarning: 示例警告行，用于验证警告着色",
|
||||
"确定性评估：2/2 成功，accounted=20，无异常",
|
||||
"模型已保存: output/hrl/ui/demo/q3.zip",
|
||||
"元数据已写入: output/hrl/ui/demo/q3.json",
|
||||
]
|
||||
return lines
|
||||
|
||||
|
||||
def demo_snapshot(*, log_tail: int = 400, scenario: str = "rising",
|
||||
done_ratio: float = 0.62, fps: float = 420.0) -> dict[str, Any]:
|
||||
"""合成一次"跑到中途"的快照（键与 TrainingJob.snapshot 一致）。
|
||||
|
||||
``done_ratio`` 控制已完成的步数比例，用来让预览更接近真实运行中状态
|
||||
（而不是刚好 100%）；``scenario`` 选择收敛判读分支；``fps`` 决定剩余
|
||||
时间估计的吞吐量级。
|
||||
"""
|
||||
from .server import describe_process, summarize_convergence
|
||||
|
||||
now = time.time()
|
||||
ratio = min(1.0, max(0.05, float(done_ratio)))
|
||||
metrics = demo_metrics(count=max(6, int(round(DEMO_STEPS * ratio))),
|
||||
total_steps=int(DEMO_CONFIG["timesteps"]),
|
||||
scenario=scenario, fps=fps)
|
||||
monitor = demo_monitor(count=max(6, int(round(60 * ratio))))
|
||||
config = dict(DEMO_CONFIG)
|
||||
lines = demo_log_lines()
|
||||
logs = (lines * 12)[-max(1, int(log_tail)):]
|
||||
status = "running"
|
||||
payload: dict[str, Any] = {
|
||||
"job_id": "job-preview",
|
||||
"status": status,
|
||||
"pid": 42424,
|
||||
"created_at": now - 186.0,
|
||||
"started_at": now - 184.0,
|
||||
"finished_at": None,
|
||||
"elapsed_s": 184.0,
|
||||
"returncode": None,
|
||||
"error": None,
|
||||
"command": list(DEMO_COMMAND),
|
||||
"model": str(config["model"]),
|
||||
"config": config,
|
||||
"log_lines": logs,
|
||||
"log_total": len(logs) * 3,
|
||||
"metrics": metrics,
|
||||
"metric_offset": 0,
|
||||
"metric_end": len(metrics),
|
||||
"metric_total": len(metrics),
|
||||
"monitor": monitor,
|
||||
"preview": True,
|
||||
"preview_scenario": scenario,
|
||||
}
|
||||
payload["convergence"] = summarize_convergence(metrics, monitor)
|
||||
payload["process"] = describe_process(payload)
|
||||
return payload
|
||||
@@ -0,0 +1,308 @@
|
||||
"""训练前端配置契约：字段元数据、分组、预设与指标字典。
|
||||
|
||||
为什么单独成模块：前端表单的取值域、单位、分组与后端 ``TrainConfig`` 的
|
||||
校验规则必须来自同一处，否则界面会允许后端拒绝的输入（或反之）。这里只
|
||||
声明"可展示"的元数据，真正的合法性判定仍由 :class:`TrainConfig.validate`
|
||||
单点负责——前端只做提前提示，不复制规则。
|
||||
|
||||
指标字典 ``METRIC_CATALOG`` 同样是唯一出处：``server.summarize_convergence``
|
||||
用它取数，``/api/train/schema`` 把它交给前端画图，避免两边各写一份键名。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
__all__ = [
|
||||
"CONFIG_FIELDS", "CONFIG_GROUPS", "PRESETS", "METRIC_CATALOG",
|
||||
"config_schema", "preset_payloads", "metric_keys",
|
||||
]
|
||||
|
||||
# 字段元数据：
|
||||
# kind number|integer|text|select|bool
|
||||
# 其余键 前端直接使用（label/unit/group/choices/min/max/step/help/advanced）
|
||||
CONFIG_FIELDS: list[dict[str, Any]] = [
|
||||
# ---------------------------------------------------------- 环境与课程
|
||||
{
|
||||
"key": "problem", "kind": "select", "group": "env",
|
||||
"label": "问题", "default": 3, "choices": [
|
||||
{"value": 3, "label": "问题 3（全向源）"},
|
||||
{"value": 4, "label": "问题 4（含定向源）"},
|
||||
],
|
||||
"help": "决定场景源类型与末段课程配置。",
|
||||
},
|
||||
{
|
||||
"key": "stage", "kind": "select", "group": "env",
|
||||
"label": "课程阶段", "default": 1, "choices": [
|
||||
{"value": 1, "label": "1 · 单源 / 5 频道 / 无噪"},
|
||||
{"value": 2, "label": "2 · 3 源 / 10 频道 / ±0.5°"},
|
||||
{"value": 3, "label": "3 · 8 源 / 16 频道 / ±1°"},
|
||||
{"value": 4, "label": "4 · 10–16 源 / 20 频道 / ±1°"},
|
||||
],
|
||||
"help": "课程递进；跨阶段续训用下方 --init 迁移权重。",
|
||||
},
|
||||
{
|
||||
"key": "config_profile", "kind": "select", "group": "env",
|
||||
"label": "配置档", "default": "production", "choices": [
|
||||
{"value": "production", "label": "production（生产保真）"},
|
||||
{"value": "coarse-pretrain", "label": "coarse-pretrain（快速粗训）"},
|
||||
],
|
||||
"help": "coarse-pretrain 显式映射为 fast 评分配置并写入模型元数据。",
|
||||
},
|
||||
{
|
||||
"key": "reward_profile", "kind": "select", "group": "env",
|
||||
"label": "奖励口径", "default": "time", "choices": [
|
||||
{"value": "time", "label": "time（P0.4 纯时间目标）"},
|
||||
{"value": "prompt", "label": "prompt（§5.2 原始塑形）"},
|
||||
],
|
||||
"help": "对照实验用；口径固化进模型元数据，不静默替换。",
|
||||
},
|
||||
{
|
||||
"key": "seed", "kind": "integer", "group": "env",
|
||||
"label": "随机种子", "default": 320001, "min": 0, "max": 2_147_483_647,
|
||||
"help": "不同种子用于多次重复训练与方差估计。",
|
||||
},
|
||||
# ---------------------------------------------------------- PPO 超参
|
||||
{
|
||||
"key": "rollout_steps", "kind": "integer", "group": "ppo",
|
||||
"label": "采样步数 n_steps", "default": 128, "min": 1, "max": 100_000,
|
||||
"help": "每次更新前每个环境采集的步数；近似一个 rollout 的宽度。",
|
||||
},
|
||||
{
|
||||
"key": "batch_size", "kind": "integer", "group": "ppo",
|
||||
"label": "批大小 batch_size", "default": 128, "min": 1, "max": 100_000,
|
||||
"help": "建议 batch_size ≤ n_steps × envs。",
|
||||
},
|
||||
{
|
||||
"key": "n_epochs", "kind": "integer", "group": "ppo",
|
||||
"label": "更新轮数 n_epochs", "default": 3, "min": 1, "max": 100,
|
||||
"help": "每批数据的 SGD 轮数；过大易使 KL 飙升。",
|
||||
},
|
||||
# ---------------------------------------------------------- 运行环境
|
||||
{
|
||||
"key": "envs", "kind": "integer", "group": "runtime",
|
||||
"label": "并行环境数", "default": 1, "min": 1, "max": 32,
|
||||
"help": "SubprocVecEnv 下每个 worker 一个进程与 Monitor 文件。",
|
||||
},
|
||||
{
|
||||
"key": "vec_env", "kind": "select", "group": "runtime",
|
||||
"label": "向量化方式", "default": "dummy", "choices": [
|
||||
{"value": "dummy", "label": "dummy（单进程，调试友好）"},
|
||||
{"value": "subproc", "label": "subproc（多进程，吞吐优先）"},
|
||||
],
|
||||
"help": "envs>1 且追求吞吐时选 subproc。",
|
||||
},
|
||||
{
|
||||
"key": "device", "kind": "select", "group": "runtime",
|
||||
"label": "计算设备", "default": "auto", "choices": [
|
||||
{"value": "auto", "label": "auto（自动选择）"},
|
||||
{"value": "cpu", "label": "cpu"},
|
||||
{"value": "cuda", "label": "cuda（NVIDIA）"},
|
||||
{"value": "mps", "label": "mps（Apple 芯片）"},
|
||||
],
|
||||
"help": "CPU 路径最稳；GPU 需与 torch 构建匹配。",
|
||||
},
|
||||
{
|
||||
"key": "local_selector", "kind": "select", "group": "runtime",
|
||||
"label": "低层执行器", "default": "geometry", "choices": [
|
||||
{"value": "geometry", "label": "geometry（几何选点，默认）"},
|
||||
{"value": "rl", "label": "rl（低层 PPO，接口预留）"},
|
||||
],
|
||||
"help": "rl 仅记录元数据，当前 HTTP 执行路径仍由高层宏动作终点决定。",
|
||||
},
|
||||
# ---------------------------------------------------------- 训练规模
|
||||
{
|
||||
"key": "timesteps", "kind": "integer", "group": "scale",
|
||||
"label": "总步数 timesteps", "default": 2048, "min": 1,
|
||||
"max": 10_000_000, "help": "本次训练的环境步总数（续训时为增量步数）。",
|
||||
},
|
||||
{
|
||||
"key": "max_wall_s", "kind": "number", "group": "scale",
|
||||
"label": "墙钟上限 (s)", "default": 1200.0, "min": 0.1, "max": 7200.0,
|
||||
"step": 30, "help": "硬上限：到点由回调主动停止并保存模型。",
|
||||
},
|
||||
{
|
||||
"key": "eval_episodes", "kind": "integer", "group": "scale",
|
||||
"label": "评估回合数", "default": 2, "min": 0, "max": 1000,
|
||||
"help": "训练结束后确定性评估；0 表示跳过。",
|
||||
},
|
||||
{
|
||||
"key": "verbose", "kind": "select", "group": "scale",
|
||||
"label": "SB3 日志级别", "default": 1, "choices": [
|
||||
{"value": 0, "label": "0 · 静默（只留 JSON 进度）"},
|
||||
{"value": 1, "label": "1 · 表格（默认）"},
|
||||
{"value": 2, "label": "2 · 调试"},
|
||||
],
|
||||
"help": "前端已解析 SB3 表格，默认 1 便于对照原始输出。",
|
||||
},
|
||||
# ---------------------------------------------------------- 模型路径
|
||||
{
|
||||
"key": "model", "kind": "text", "group": "paths",
|
||||
"label": "模型输出路径", "default": "output/hrl/ui/q3.zip",
|
||||
"help": "相对仓库根；同目录会写入 .json 元数据与 Monitor CSV。",
|
||||
},
|
||||
{
|
||||
"key": "init", "kind": "text", "group": "paths", "optional": True,
|
||||
"label": "初始权重 --init", "default": None,
|
||||
"help": "留空=从零；填上一阶段 .zip 即课程迁移（步数计数重置）。",
|
||||
},
|
||||
{
|
||||
"key": "local_model", "kind": "text", "group": "paths", "optional": True,
|
||||
"label": "低层模型 --local-model", "default": None,
|
||||
"advanced": True,
|
||||
"help": "仅 local_selector=rl 时有意义。",
|
||||
},
|
||||
{
|
||||
"key": "resume", "kind": "bool", "group": "paths",
|
||||
"label": "续训 --resume", "default": False,
|
||||
"help": "从模型输出路径已有 .zip 继续，timesteps 按增量理解。",
|
||||
},
|
||||
{
|
||||
"key": "force", "kind": "bool", "group": "paths",
|
||||
"label": "覆盖已有模型 --force", "default": False,
|
||||
"help": "输出路径已存在时必须勾选，否则训练脚本会拒绝覆盖。与续训互斥。",
|
||||
},
|
||||
]
|
||||
|
||||
CONFIG_GROUPS: list[dict[str, str]] = [
|
||||
{"key": "env", "label": "环境与课程", "hint": "题目、课程阶段与奖励口径"},
|
||||
{"key": "ppo", "label": "PPO 超参数", "hint": "采样宽度与更新强度"},
|
||||
{"key": "runtime", "label": "运行环境", "hint": "并行、设备与低层执行器"},
|
||||
{"key": "scale", "label": "训练规模", "hint": "步数、墙钟上限与评估"},
|
||||
{"key": "paths", "label": "模型与迁移", "hint": "产物路径、续训与课程迁移"},
|
||||
]
|
||||
|
||||
# 预设 = 只覆盖差异字段；后端 from_payload 会补齐其余默认值。
|
||||
PRESETS: list[dict[str, Any]] = [
|
||||
{
|
||||
"key": "smoke",
|
||||
"label": "冒烟（约 1 分钟）",
|
||||
"note": "64 步、CPU、无评估——只验证链路与界面。",
|
||||
"payload": {
|
||||
"problem": 3, "stage": 1, "timesteps": 64, "seed": 1,
|
||||
"envs": 1, "vec_env": "dummy", "device": "cpu",
|
||||
"config_profile": "coarse-pretrain", "reward_profile": "time",
|
||||
"model": "output/hrl/ui/smoke/q3.zip", "eval_episodes": 0,
|
||||
"rollout_steps": 32, "batch_size": 32, "n_epochs": 2,
|
||||
"max_wall_s": 180.0, "verbose": 1, "force": True,
|
||||
},
|
||||
},
|
||||
{
|
||||
"key": "local-check",
|
||||
"label": "本地验证（约 10 分钟）",
|
||||
"note": "stage 1 单源课程 2048 步 + 2 回合评估。",
|
||||
"payload": {
|
||||
"problem": 3, "stage": 1, "timesteps": 2048, "seed": 320001,
|
||||
"envs": 1, "vec_env": "dummy", "device": "auto",
|
||||
"config_profile": "production", "reward_profile": "time",
|
||||
"model": "output/hrl/ui/local/q3.zip", "eval_episodes": 2,
|
||||
"rollout_steps": 128, "batch_size": 128, "n_epochs": 3,
|
||||
"max_wall_s": 900.0, "verbose": 1, "force": True,
|
||||
},
|
||||
},
|
||||
{
|
||||
"key": "q3-main",
|
||||
"label": "Q3 主训练",
|
||||
"note": "stage 3 八源课程，4 进程并行；建议在训练机上跑。",
|
||||
"payload": {
|
||||
"problem": 3, "stage": 3, "timesteps": 200000, "seed": 320001,
|
||||
"envs": 4, "vec_env": "subproc", "device": "auto",
|
||||
"config_profile": "production", "reward_profile": "time",
|
||||
"model": "output/hrl/ui/q3-main/q3.zip", "eval_episodes": 5,
|
||||
"rollout_steps": 256, "batch_size": 256, "n_epochs": 10,
|
||||
"max_wall_s": 7200.0, "verbose": 1,
|
||||
},
|
||||
},
|
||||
{
|
||||
"key": "q4-transfer",
|
||||
"label": "Q4 课程迁移",
|
||||
"note": "从 Q3 权重迁移到问题 4 末段课程（含定向源）。",
|
||||
"payload": {
|
||||
"problem": 4, "stage": 4, "timesteps": 120000, "seed": 420001,
|
||||
"envs": 4, "vec_env": "subproc", "device": "auto",
|
||||
"config_profile": "production", "reward_profile": "time",
|
||||
"model": "output/hrl/ui/q4-main/q4.zip",
|
||||
"init": "output/hrl/ui/q3-main/q3.zip", "eval_episodes": 5,
|
||||
"rollout_steps": 256, "batch_size": 256, "n_epochs": 10,
|
||||
"max_wall_s": 7200.0, "verbose": 1,
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
# 指标字典：键来自训练脚本 --progress-json 的单行 JSON（短名）与 SB3 表格
|
||||
# （归一化去前缀）。前端按此顺序绘图，收敛判据在 server.summarize_convergence。
|
||||
METRIC_CATALOG: list[dict[str, Any]] = [
|
||||
{
|
||||
"key": "ep_rew_mean", "label": "平均回合奖励", "role": "primary",
|
||||
"color": "#4f8cff", "format": "number", "better": "up",
|
||||
"help": "最近若干回合的平均回报；主要收敛判据。",
|
||||
},
|
||||
{
|
||||
"key": "ep_len_mean", "label": "平均回合长度", "role": "secondary",
|
||||
"color": "#38bdf8", "format": "number", "better": "neutral",
|
||||
"help": "回合步数；与奖励同时上升通常代表策略更完整而非更快。",
|
||||
},
|
||||
{
|
||||
"key": "approx_kl", "label": "近似 KL", "role": "stability",
|
||||
"color": "#f59e0b", "format": "sci", "better": "down",
|
||||
"help": "策略更新幅度；> 0.05 视为更新过猛。",
|
||||
},
|
||||
{
|
||||
"key": "explained_variance", "label": "解释方差", "role": "stability",
|
||||
"color": "#34d399", "format": "number", "better": "up",
|
||||
"help": "价值函数拟合优度；< 0 表示还不如均值预测。",
|
||||
},
|
||||
{
|
||||
"key": "value_loss", "label": "价值损失", "role": "stability",
|
||||
"color": "#f87171", "format": "sci", "better": "down",
|
||||
"help": "critic 拟合误差。",
|
||||
},
|
||||
{
|
||||
"key": "policy_gradient_loss", "label": "策略梯度损失", "role": "stability",
|
||||
"color": "#c084fc", "format": "sci", "better": "neutral",
|
||||
"help": "actor 的代理损失；量级小且符号可变属正常。",
|
||||
},
|
||||
{
|
||||
"key": "entropy_loss", "label": "策略熵", "role": "exploration",
|
||||
"color": "#fbbf24", "format": "number", "better": "neutral",
|
||||
"help": "探索度；若长期贴近 0 说明策略过早确定。",
|
||||
},
|
||||
{
|
||||
"key": "n_updates", "label": "更新次数", "role": "throughput",
|
||||
"color": "#93a1bd", "format": "integer", "better": "up",
|
||||
"help": "已完成的 PPO 更新轮数。",
|
||||
},
|
||||
{
|
||||
"key": "fps", "label": "吞吐 FPS", "role": "throughput",
|
||||
"color": "#22d3ee", "format": "number", "better": "up",
|
||||
"help": "每秒环境步数，反映硬件与并行配置是否吃满。",
|
||||
},
|
||||
]
|
||||
|
||||
_METRIC_BY_KEY = {item["key"]: item for item in METRIC_CATALOG}
|
||||
|
||||
|
||||
def metric_keys() -> list[str]:
|
||||
return [item["key"] for item in METRIC_CATALOG]
|
||||
|
||||
|
||||
def metric_meta(key: str) -> dict[str, Any]:
|
||||
return _METRIC_BY_KEY.get(key, {"key": key, "label": key, "role": "other",
|
||||
"color": "#93a1bd", "format": "number",
|
||||
"better": "neutral", "help": ""})
|
||||
|
||||
|
||||
def config_schema() -> dict[str, Any]:
|
||||
"""返回给前端的完整配置契约（字段 + 分组 + 预设 + 指标字典）。"""
|
||||
return {
|
||||
"fields": CONFIG_FIELDS,
|
||||
"groups": CONFIG_GROUPS,
|
||||
"presets": [{"key": item["key"], "label": item["label"],
|
||||
"note": item["note"], "payload": dict(item["payload"])}
|
||||
for item in PRESETS],
|
||||
"metrics": METRIC_CATALOG,
|
||||
}
|
||||
|
||||
|
||||
def preset_payloads() -> dict[str, dict[str, Any]]:
|
||||
return {item["key"]: dict(item["payload"]) for item in PRESETS}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,756 @@
|
||||
/* 训练前端主逻辑：纯原生 JS，无外部依赖。
|
||||
职责分工：
|
||||
- 表单由 /api/train/schema 的字段元数据生成，不在前端复制校验规则；
|
||||
- 训练数值全部取自 /api/train/state 快照（服务端已算好进度、吞吐、
|
||||
收敛判读），前端只做格式化与渲染，避免两套算法对不上；
|
||||
- 指标增量拉取（after_metric 游标），日志按尾部行数拉取。 */
|
||||
"use strict";
|
||||
|
||||
const $ = (id) => document.getElementById(id);
|
||||
const REFRESH_MS = 1500;
|
||||
// 与服务端 MAX_METRICS 对齐：本地缓存上限，避免长时间训练把内存吃满
|
||||
const METRIC_LIMIT = 5000;
|
||||
|
||||
const state = {
|
||||
schema: null,
|
||||
preview: false, // 离线预览：不再启动进程，只渲染示例快照
|
||||
polling: false,
|
||||
saving: false, // 配置表单是否处于"训练中"锁定态
|
||||
metrics: [], // 本地累积的进度点（按 timesteps 递增）
|
||||
metricCache: [], // metrics 的同一个数组（显式持有，避免 slice/concat 时丢引用）
|
||||
metricNext: 0, // 本地已吃到的下一个下标（= 服务端下次应返回的 offset）
|
||||
metricTotal: 0, // 服务端累计进度点总数（增量拉取的游标）
|
||||
chartKey: "ep_rew_mean",
|
||||
jobKey: null, // job_id 变化时清空本地缓存
|
||||
lastLogSignature: "",
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------- 工具
|
||||
function escapeHtml(value) {
|
||||
return String(value == null ? "" : value)
|
||||
.replace(/[&<>"']/g, (c) => ({ "&": "&", "<": "<", ">": ">",
|
||||
'"': """, "'": "'" }[c]));
|
||||
}
|
||||
function fmtNumber(value, digits) {
|
||||
if (value === null || value === undefined || value === "" || Number.isNaN(Number(value))) return "—";
|
||||
const num = Number(value);
|
||||
const abs = Math.abs(num);
|
||||
if (abs !== 0 && (abs < 1e-3 || abs >= 1e5)) return num.toExponential(2);
|
||||
const d = digits === undefined ? (abs >= 100 ? 1 : abs >= 1 ? 2 : 4) : digits;
|
||||
return num.toFixed(d);
|
||||
}
|
||||
function fmtInt(value) {
|
||||
if (value === null || value === undefined || Number.isNaN(Number(value))) return "—";
|
||||
return Math.round(Number(value)).toLocaleString("en-US");
|
||||
}
|
||||
function fmtDuration(seconds) {
|
||||
if (seconds === null || seconds === undefined || !isFinite(seconds) || seconds < 0) return "—";
|
||||
const s = Math.round(seconds);
|
||||
const h = Math.floor(s / 3600), m = Math.floor((s % 3600) / 60), sec = s % 60;
|
||||
if (h > 0) return `${h}h${String(m).padStart(2, "0")}m`;
|
||||
if (m > 0) return `${m}m${String(sec).padStart(2, "0")}s`;
|
||||
return `${sec}s`;
|
||||
}
|
||||
function fmtClock(epochSeconds) {
|
||||
if (!epochSeconds) return "—";
|
||||
const d = new Date(epochSeconds * 1000);
|
||||
return `${String(d.getHours()).padStart(2, "0")}:${String(d.getMinutes()).padStart(2, "0")}:${String(d.getSeconds()).padStart(2, "0")}`;
|
||||
}
|
||||
function chip(id, label, value, tone) {
|
||||
const el = $(id);
|
||||
if (!el) return;
|
||||
el.className = `chip ${tone || "gray"}`;
|
||||
el.innerHTML = `${label}<b>${escapeHtml(value)}</b>`;
|
||||
}
|
||||
function setText(id, text) { const el = $(id); if (el) el.textContent = text; }
|
||||
|
||||
// ---------------------------------------------------------------- 启动
|
||||
async function boot() {
|
||||
try {
|
||||
const res = await fetch("/api/train/schema");
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
state.schema = await res.json();
|
||||
} catch (err) {
|
||||
state.schema = { fields: [], groups: [], presets: [], metrics: [] };
|
||||
showBanner("bad", `无法读取配置契约（${err.message}）：请确认已通过 python -m mcm26b.training_ui 启动前端。`);
|
||||
}
|
||||
buildForm();
|
||||
await poll();
|
||||
bindChartInteractions();
|
||||
setInterval(poll, REFRESH_MS);
|
||||
}
|
||||
|
||||
function showBanner(tone, html) {
|
||||
const el = $("banner");
|
||||
if (!html) { el.classList.add("hidden"); el.innerHTML = ""; return; }
|
||||
el.className = `banner ${tone || ""}`;
|
||||
el.innerHTML = html;
|
||||
el.classList.remove("hidden");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 表单
|
||||
function buildForm() {
|
||||
const presets = $("presets");
|
||||
presets.innerHTML = "";
|
||||
for (const preset of state.schema.presets || []) {
|
||||
const btn = document.createElement("button");
|
||||
btn.type = "button";
|
||||
btn.className = "preset";
|
||||
btn.innerHTML = `${escapeHtml(preset.label)}<span class="note">${escapeHtml(preset.note || "")}</span>`;
|
||||
btn.onclick = () => { applyValues(preset.payload); $("form-error").textContent = ""; };
|
||||
presets.appendChild(btn);
|
||||
}
|
||||
|
||||
const wrap = $("form-groups");
|
||||
wrap.innerHTML = "";
|
||||
for (const group of state.schema.groups || []) {
|
||||
const box = document.createElement("fieldset");
|
||||
box.className = "group";
|
||||
const fields = (state.schema.fields || []).filter((f) => f.group === group.key);
|
||||
box.innerHTML = `<legend>${escapeHtml(group.label)}</legend>` +
|
||||
`<p class="ghint">${escapeHtml(group.hint || "")}</p>`;
|
||||
for (const field of fields) box.appendChild(buildField(field));
|
||||
wrap.appendChild(box);
|
||||
}
|
||||
applyValues({});
|
||||
}
|
||||
|
||||
function buildField(field) {
|
||||
const row = document.createElement("div");
|
||||
row.className = "field";
|
||||
row.id = `field-${field.key}`;
|
||||
const label = document.createElement("label");
|
||||
label.className = "flabel";
|
||||
label.innerHTML = `<span>${escapeHtml(field.label)}</span>` +
|
||||
(field.unit ? `<span class="unit">${escapeHtml(field.unit)}</span>` : "");
|
||||
let control;
|
||||
if (field.kind === "select") {
|
||||
control = document.createElement("select");
|
||||
for (const choice of field.choices || []) {
|
||||
const option = document.createElement("option");
|
||||
option.value = String(choice.value);
|
||||
option.textContent = choice.label;
|
||||
control.appendChild(option);
|
||||
}
|
||||
} else if (field.kind === "bool") {
|
||||
control = document.createElement("input");
|
||||
control.type = "checkbox";
|
||||
row.classList.add("check");
|
||||
} else {
|
||||
control = document.createElement("input");
|
||||
control.type = field.kind === "text" ? "text" : "number";
|
||||
if (field.kind === "number") control.step = field.step || "any";
|
||||
if (field.min !== undefined) control.min = field.min;
|
||||
if (field.max !== undefined) control.max = field.max;
|
||||
}
|
||||
control.id = `cfg-${field.key}`;
|
||||
control.dataset.key = field.key;
|
||||
control.addEventListener("input", () => { row.classList.add("dirty"); row.classList.remove("invalid"); });
|
||||
row.appendChild(label);
|
||||
row.appendChild(control);
|
||||
if (field.help) {
|
||||
const help = document.createElement("span");
|
||||
help.className = "fhelp";
|
||||
help.textContent = field.help;
|
||||
row.appendChild(help);
|
||||
}
|
||||
return row;
|
||||
}
|
||||
|
||||
function setControl(field, value) {
|
||||
const control = $(`cfg-${field.key}`);
|
||||
if (!control) return;
|
||||
if (field.kind === "bool") { control.checked = Boolean(value); return; }
|
||||
if (value === null || value === undefined) { control.value = ""; return; }
|
||||
control.value = String(value);
|
||||
}
|
||||
|
||||
function applyValues(values) {
|
||||
for (const field of state.schema.fields || []) {
|
||||
const has = Object.prototype.hasOwnProperty.call(values, field.key);
|
||||
setControl(field, has ? values[field.key] : field.default);
|
||||
const row = $(`field-${field.key}`);
|
||||
if (row) row.classList.remove("dirty", "invalid");
|
||||
}
|
||||
}
|
||||
|
||||
function collectValues() {
|
||||
const out = {};
|
||||
for (const field of state.schema.fields || []) {
|
||||
const control = $(`cfg-${field.key}`);
|
||||
if (!control) continue;
|
||||
if (field.kind === "bool") { out[field.key] = control.checked; continue; }
|
||||
const raw = String(control.value || "").trim();
|
||||
if (raw === "") { out[field.key] = null; continue; }
|
||||
if (field.kind === "integer") out[field.key] = Number.parseInt(raw, 10);
|
||||
else if (field.kind === "number") out[field.key] = Number.parseFloat(raw);
|
||||
else if (field.kind === "select") {
|
||||
const choices = field.choices || [];
|
||||
const match = choices.find((c) => String(c.value) === raw);
|
||||
out[field.key] = match ? match.value : raw;
|
||||
} else out[field.key] = raw;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/* 与后端 validate() 对齐的前置提示：只做提示，不阻止提交后端的权威判定 */
|
||||
function validateLocally(values) {
|
||||
const problems = [];
|
||||
for (const field of state.schema.fields || []) {
|
||||
const value = values[field.key];
|
||||
if (value === null || value === undefined) continue;
|
||||
if (field.kind === "integer" || field.kind === "number") {
|
||||
if (!isFinite(value)) { problems.push(`${field.label} 不是有效数字`); markInvalid(field.key, true); continue; }
|
||||
if (field.min !== undefined && value < field.min) { problems.push(`${field.label} 不能小于 ${field.min}`); markInvalid(field.key, true); continue; }
|
||||
if (field.max !== undefined && value > field.max) { problems.push(`${field.label} 不能大于 ${field.max}`); markInvalid(field.key, true); continue; }
|
||||
markInvalid(field.key, false);
|
||||
}
|
||||
}
|
||||
if (values.rollout_steps && values.timesteps && values.rollout_steps > values.timesteps) {
|
||||
problems.push("采样步数 n_steps 不能大于总步数 timesteps");
|
||||
markInvalid("rollout_steps", true);
|
||||
}
|
||||
if (values.batch_size && values.rollout_steps && values.envs
|
||||
&& values.batch_size > values.rollout_steps * values.envs) {
|
||||
problems.push("批大小 batch_size 大于 n_steps × envs，PPO 每轮数据不足以分批");
|
||||
markInvalid("batch_size", true);
|
||||
}
|
||||
return problems;
|
||||
}
|
||||
|
||||
function markInvalid(key, bad) {
|
||||
const row = $(`field-${key}`);
|
||||
if (row) row.classList.toggle("invalid", Boolean(bad));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 轮询
|
||||
async function poll() {
|
||||
if (state.polling) return;
|
||||
if (state.preview) return;
|
||||
state.polling = true;
|
||||
try {
|
||||
const query = new URLSearchParams({
|
||||
log_tail: $("log-tail").value || "400",
|
||||
metric_limit: String(METRIC_LIMIT),
|
||||
});
|
||||
if (state.metricTotal > 0) query.set("after_metric", String(state.metricTotal));
|
||||
const res = await fetch(`/api/train/state?${query.toString()}`);
|
||||
const payload = await res.json();
|
||||
if (!res.ok) throw new Error(payload.error || `HTTP ${res.status}`);
|
||||
chip("chip-conn", "连接", "已连接", "ok");
|
||||
render(payload.job);
|
||||
} catch (err) {
|
||||
chip("chip-conn", "连接", "断开", "bad");
|
||||
} finally {
|
||||
state.polling = false;
|
||||
}
|
||||
}
|
||||
|
||||
/* 合并一次快照返回的进度点。服务端用 metric_offset 给出这批点的绝对起点，
|
||||
客户端只需与"本地已吃到的下一个下标"（metricNext）对账：
|
||||
- offset === metricNext：正常续接（增量拉取），concat；
|
||||
- offset > metricNext：本地缓存与这批点之间缺了若干点（例如页面刷新后
|
||||
服务端窗口已前移），保留本地旧点会让曲线出现空洞，因此整体替换；
|
||||
- offset < metricNext：本地缓存比服务端更新（服务端重启/计数回退），
|
||||
忽略这批过期点，不动本地缓存。
|
||||
判据是"绝对下标"而不是长度：游标是服务端总数，本地可能被裁剪得更短，
|
||||
拿长度比较会把同一批点重复追加（本函数注释即为此 bug 的回归说明）。
|
||||
下标必须显式转 Number：JSON 里若出现字符串形式的数字，JS 的 >= 会退化成
|
||||
字典序比较（"14" >= "014" 为 false），静默走错分支。 */
|
||||
function absorbMetrics(job) {
|
||||
const incoming = Array.isArray(job.metrics) ? job.metrics : [];
|
||||
const rawOffset = Number(job.metric_offset);
|
||||
const offset = Number.isFinite(rawOffset) ? rawOffset : null;
|
||||
const cache = state.metricCache || [];
|
||||
if (incoming.length && offset !== null) {
|
||||
if (!cache.length || offset > state.metricNext) {
|
||||
// 首批，或与本地之间有缺口（服务端窗口前移）：整体替换，避免曲线断档
|
||||
state.metricCache = incoming.slice();
|
||||
state.metricNext = offset + incoming.length;
|
||||
} else if (offset === state.metricNext) {
|
||||
state.metricCache = cache.concat(incoming);
|
||||
state.metricNext = offset + incoming.length;
|
||||
} else {
|
||||
return; // 过期批次：本地已有更新的点，直接忽略
|
||||
}
|
||||
}
|
||||
if (state.metricCache.length > METRIC_LIMIT) {
|
||||
state.metricCache = state.metricCache.slice(state.metricCache.length - METRIC_LIMIT);
|
||||
}
|
||||
state.metrics = state.metricCache;
|
||||
const total = Number(job.metric_total);
|
||||
if (Number.isFinite(total)) state.metricTotal = total;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 渲染
|
||||
function render(job) {
|
||||
if (!job) {
|
||||
renderIdle();
|
||||
return;
|
||||
}
|
||||
if (job.job_id !== state.jobKey) {
|
||||
state.jobKey = job.job_id;
|
||||
state.metrics = [];
|
||||
state.metricTotal = 0;
|
||||
state.lastLogSignature = "";
|
||||
}
|
||||
absorbMetrics(job);
|
||||
|
||||
renderProcess(job);
|
||||
renderConvergence(job);
|
||||
renderCharts();
|
||||
renderLog(job);
|
||||
renderEpisodes(job);
|
||||
renderBanner(job);
|
||||
renderJobLine(job);
|
||||
}
|
||||
|
||||
function renderIdle() {
|
||||
chip("chip-process", "进程", "未启动", "gray");
|
||||
chip("chip-progress", "进度", "—", "gray");
|
||||
chip("chip-elapsed", "墙钟", "—", "gray");
|
||||
chip("chip-device", "设备", "—", "gray");
|
||||
setText("process-hint", "启动任务后显示子进程与步进进度");
|
||||
setText("steps-done", "0"); setText("steps-total", "0");
|
||||
setText("progress-percent", "0.0%");
|
||||
$("bar-fill").style.width = "0%";
|
||||
setText("eta", "—"); setText("eta-clock", "—"); setText("metric-total", "0");
|
||||
for (const id of ["stat-pid", "stat-fps", "stat-elapsed", "stat-rc"]) setText(id, "—");
|
||||
setText("stat-logs", "0");
|
||||
$("timeline").innerHTML = "";
|
||||
setText("command-preview", "");
|
||||
chip("chip-converge", "判读", "等待数据", "gray");
|
||||
$("converge-notes").innerHTML = "";
|
||||
$("metric-grid").innerHTML = '<p class="hint">尚未启动训练，暂无参数。</p>';
|
||||
setFormLocked(false);
|
||||
}
|
||||
|
||||
function renderJobLine(job) {
|
||||
const cfg = job.config || {};
|
||||
const parts = [
|
||||
`任务 ${job.job_id}`, `问题 ${cfg.problem}`, `阶段 ${cfg.stage}`,
|
||||
`口径 ${cfg.reward_profile}/${cfg.config_profile}`,
|
||||
`seed ${cfg.seed}`, `envs ${cfg.envs}·${cfg.vec_env}`,
|
||||
];
|
||||
setText("job-line", parts.join(" · "));
|
||||
chip("chip-device", "设备", `${cfg.device || "?"}`, "cyan");
|
||||
}
|
||||
|
||||
function renderProcess(job) {
|
||||
const process = job.process || {};
|
||||
const tone = process.tone || "gray";
|
||||
chip("chip-process", "进程", process.label || job.status, tone);
|
||||
chip("chip-elapsed", "墙钟", fmtDuration(process.elapsed_s), tone === "gray" ? "gray" : "cyan");
|
||||
|
||||
const percent = process.percent;
|
||||
setText("chip-progress-value", "");
|
||||
chip("chip-progress", "进度",
|
||||
percent === null || percent === undefined ? "—" : `${percent.toFixed(1)}%`,
|
||||
percent !== null && percent >= 100 ? "ok" : "cyan");
|
||||
setText("steps-done", fmtInt(process.done_steps));
|
||||
setText("steps-total", fmtInt(process.total_steps));
|
||||
setText("progress-percent", percent === null || percent === undefined ? "—" : `${percent.toFixed(2)}%`);
|
||||
const fill = $("bar-fill");
|
||||
fill.style.width = `${percent === null || percent === undefined ? 0 : percent}%`;
|
||||
fill.className = "bar-fill" + (tone === "bad" ? " bad" : tone === "warn" ? " warn" : tone === "ok" && percent >= 100 ? " ok" : "");
|
||||
setText("process-hint", process.hint || "");
|
||||
setText("eta", fmtDuration(process.remaining_s));
|
||||
setText("eta-clock", fmtClock(process.eta_epoch_s));
|
||||
setText("metric-total", fmtInt(process.metric_total));
|
||||
|
||||
setText("stat-pid", process.terminal ? "已退出" : (job.pid ? `PID ${job.pid}` : "—"));
|
||||
setText("stat-pgid", process.terminal ? "已回收" : "独立会话");
|
||||
const fps = state.metrics.length ? lastFinite(state.metrics, "fps") : null;
|
||||
setText("stat-fps", fps === null ? "—" : fmtNumber(fps, 1));
|
||||
setText("stat-elapsed", fmtDuration(process.elapsed_s));
|
||||
const rc = $("stat-rc");
|
||||
rc.textContent = job.returncode === null || job.returncode === undefined ? "—" : String(job.returncode);
|
||||
rc.className = "v" + (job.returncode === 0 ? " ok" : job.returncode ? " bad" : "");
|
||||
setText("stat-logs", `${fmtInt(process.log_total)} / ${fmtInt(process.log_capacity)}`);
|
||||
|
||||
renderTimeline(process);
|
||||
setText("command-preview", job.command ? `$ ${job.command.join(" ")}` : "");
|
||||
setFormLocked(job.status === "running" || job.status === "created" || job.status === "stopping");
|
||||
$("btn-stop").disabled = !(job.status === "running" || job.status === "created");
|
||||
}
|
||||
|
||||
function renderTimeline(process) {
|
||||
const wrap = $("timeline");
|
||||
const nodes = process.timeline || [];
|
||||
if (!nodes.length) { wrap.innerHTML = ""; return; }
|
||||
const active = nodes.findIndex((n) => n.state === "active");
|
||||
if (wrap.dataset.signature !== nodes.map((n) => n.label).join("|")) {
|
||||
wrap.innerHTML = "";
|
||||
nodes.forEach((node, index) => {
|
||||
if (index > 0) {
|
||||
const link = document.createElement("span");
|
||||
link.className = "link";
|
||||
link.dataset.link = String(index);
|
||||
wrap.appendChild(link);
|
||||
}
|
||||
const el = document.createElement("span");
|
||||
el.className = "node";
|
||||
el.dataset.node = node.key;
|
||||
el.innerHTML = `<span class="dot"></span><span class="txt">${escapeHtml(node.label)}</span>`;
|
||||
wrap.appendChild(el);
|
||||
});
|
||||
wrap.dataset.signature = nodes.map((n) => n.label).join("|");
|
||||
}
|
||||
nodes.forEach((node, index) => {
|
||||
const el = wrap.querySelector(`[data-node="${node.key}"]`);
|
||||
if (!el) return;
|
||||
let cls = "node";
|
||||
if (node.state === "done") cls += " done";
|
||||
else if (node.state === "active") cls += " active";
|
||||
if (node.state === "active" && process.tone === "bad") cls += " failed";
|
||||
el.className = cls;
|
||||
const link = wrap.querySelector(`[data-link="${index}"]`);
|
||||
if (link) link.className = "link" + (index <= active ? " done" : "");
|
||||
});
|
||||
}
|
||||
|
||||
function setFormLocked(locked) {
|
||||
state.saving = locked;
|
||||
for (const field of state.schema.fields || []) {
|
||||
const control = $(`cfg-${field.key}`);
|
||||
if (control) control.disabled = locked;
|
||||
}
|
||||
$("btn-start").disabled = locked;
|
||||
$("btn-start").textContent = locked ? "训练中…" : "开始训练";
|
||||
}
|
||||
|
||||
function renderBanner(job) {
|
||||
const process = job.process || {};
|
||||
if (state.preview) {
|
||||
showBanner("warn", "离线预览模式：正在渲染示例快照，<b>不会</b>启动训练进程。点「刷新」可返回实时轮询。");
|
||||
return;
|
||||
}
|
||||
if (job.status === "failed") {
|
||||
showBanner("bad", `训练异常结束（退出码 <b>${job.returncode === null ? "?" : job.returncode}</b>）：${escapeHtml(job.error || "见日志尾部")}。`);
|
||||
} else if (job.status === "stopped") {
|
||||
showBanner("warn", "已按请求停止训练（SIGTERM 独立进程组），本轮模型可能未保存。");
|
||||
} else if (job.status === "exited") {
|
||||
showBanner("ok", "训练与评估流程正常结束，模型与元数据已写入输出目录。");
|
||||
} else if (process.remaining_s !== null && process.remaining_s !== undefined
|
||||
&& process.eta_epoch_s === null && job.status === "running") {
|
||||
showBanner("warn", `尚未产出可估算吞吐的进度点，剩余时间按墙钟上限回退估计：<b>${fmtDuration(process.remaining_s)}</b>。`);
|
||||
} else {
|
||||
showBanner("", "");
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 收敛
|
||||
function renderConvergence(job) {
|
||||
const convergence = job.convergence || {};
|
||||
chip("chip-converge", "判读", convergence.status || "等待数据", convergence.tone || "gray");
|
||||
const notes = convergence.notes || [];
|
||||
$("converge-notes").innerHTML = notes.length
|
||||
? notes.map((note) => `<li>${escapeHtml(note)}</li>`).join("")
|
||||
: "<li>暂无判读说明。</li>";
|
||||
|
||||
const grid = $("metric-grid");
|
||||
grid.innerHTML = "";
|
||||
const catalog = state.schema.metrics || [];
|
||||
for (const metric of catalog) {
|
||||
const points = seriesOf(metric.key);
|
||||
const values = points.map((p) => p[1]);
|
||||
const value = values.length ? values[values.length - 1] : null;
|
||||
const prev = values.length > 1 ? values[values.length - 2] : null;
|
||||
const delta = value !== null && prev !== null ? value - prev : null;
|
||||
const card = document.createElement("div");
|
||||
card.className = "metric-card";
|
||||
const alert = isAlert(metric.key, value);
|
||||
if (alert) card.classList.add("alert");
|
||||
const trendClass = delta === null || Math.abs(delta) < 1e-12 ? "flat"
|
||||
: delta > 0 ? "up" : "down";
|
||||
const arrow = delta === null || Math.abs(delta) < 1e-12 ? "→" : delta > 0 ? "▲" : "▼";
|
||||
card.innerHTML =
|
||||
`<div class="mhead"><span class="mname">${escapeHtml(metric.label)}</span>` +
|
||||
`<span class="mvalue">${escapeHtml(fmtValue(metric, value))}</span></div>` +
|
||||
`<canvas class="spark" data-spark="${metric.key}"></canvas>` +
|
||||
`<div class="mfoot"><span class="trend ${trendClass}" data-delta="${metric.key}">${arrow} ${escapeHtml(delta === null ? "—" : fmtValue(metric, delta))}</span>` +
|
||||
`<span>${points.length} 点</span></div>`;
|
||||
if (alert) card.title = alert;
|
||||
grid.appendChild(card);
|
||||
const canvas = card.querySelector("canvas");
|
||||
if (canvas && values.length >= 2) {
|
||||
window.Charts.sparkline(canvas, values.slice(-120), { color: metric.color });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function isAlert(key, value) {
|
||||
if (value === null || value === undefined) return null;
|
||||
if (key === "approx_kl" && value > 0.05) return "近似 KL 超过 0.05：策略更新幅度过大";
|
||||
if (key === "explained_variance" && value < 0) return "解释方差为负：价值函数尚未拟合";
|
||||
if (key === "entropy_loss" && Math.abs(value) < 1e-3) return "策略熵接近 0：探索可能过早停止";
|
||||
return null;
|
||||
}
|
||||
|
||||
function fmtValue(metric, value) {
|
||||
if (value === null || value === undefined || !isFinite(value)) return "—";
|
||||
if (metric.format === "integer") return fmtInt(value);
|
||||
return fmtNumber(value);
|
||||
}
|
||||
|
||||
function seriesOf(key) {
|
||||
const points = [];
|
||||
for (const row of state.metrics) {
|
||||
const value = Number(row[key]);
|
||||
if (!isFinite(value)) continue;
|
||||
const x = Number(row.timesteps ?? row.total_timesteps ?? points.length + 1);
|
||||
points.push([isFinite(x) ? x : points.length + 1, value]);
|
||||
}
|
||||
return points;
|
||||
}
|
||||
function lastFinite(rows, key) {
|
||||
for (let i = rows.length - 1; i >= 0; i--) {
|
||||
const value = Number(rows[i][key]);
|
||||
if (isFinite(value)) return value;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 图表
|
||||
function renderCharts() {
|
||||
const catalog = state.schema.metrics || [];
|
||||
const tabs = $("chart-tabs");
|
||||
if (tabs.dataset.signature !== catalog.map((m) => m.key).join("|")) {
|
||||
tabs.innerHTML = "";
|
||||
for (const metric of catalog) {
|
||||
const btn = document.createElement("button");
|
||||
btn.type = "button";
|
||||
btn.className = "chart-tab";
|
||||
btn.dataset.key = metric.key;
|
||||
btn.textContent = metric.label;
|
||||
btn.onclick = () => { state.chartKey = metric.key; renderCharts(); };
|
||||
tabs.appendChild(btn);
|
||||
}
|
||||
tabs.dataset.signature = catalog.map((m) => m.key).join("|");
|
||||
}
|
||||
for (const btn of tabs.querySelectorAll(".chart-tab")) {
|
||||
btn.classList.toggle("active", btn.dataset.key === state.chartKey);
|
||||
}
|
||||
|
||||
const windowSize = Number($("chart-window").value || 0);
|
||||
const primary = catalog.find((m) => m.role === "primary") || { key: "ep_rew_mean", label: "奖励", color: "#4f8cff" };
|
||||
const rewardPoints = slice(seriesOf(primary.key), windowSize);
|
||||
const layout = window.Charts.line($("chart-reward"), {
|
||||
series: [{ name: primary.label, points: rewardPoints, color: primary.color, fill: true }],
|
||||
xFormat: (v) => `${fmtInt(v)}步`,
|
||||
emptyText: "等待首个 rollout 的 ep_rew_mean……",
|
||||
});
|
||||
setText("chart-title", primary.label);
|
||||
setText("chart-sub", describeSeries(rewardPoints, primary));
|
||||
|
||||
const metric = catalog.find((m) => m.key === state.chartKey) || primary;
|
||||
const metricPoints = slice(seriesOf(metric.key), windowSize);
|
||||
const refs = [];
|
||||
if (metric.key === "approx_kl") refs.push({ value: 0.05, color: "#f87171", label: "过猛阈值 0.05" });
|
||||
if (metric.key === "explained_variance") refs.push({ value: 0, color: "#93a1bd", label: "0（不如均值预测）" });
|
||||
const layout2 = window.Charts.line($("chart-metric"), {
|
||||
series: [{ name: metric.label, points: metricPoints, color: metric.color, fill: true }],
|
||||
refs: refs,
|
||||
xFormat: (v) => `${fmtInt(v)}步`,
|
||||
emptyText: `等待 ${metric.key} 数据……`,
|
||||
});
|
||||
setText("chart2-title", metric.label);
|
||||
setText("chart2-sub", describeSeries(metricPoints, metric));
|
||||
state.layouts = { reward: layout, metric: layout2, metricKey: metric.key, primary: primary };
|
||||
}
|
||||
|
||||
function slice(points, size) {
|
||||
if (!size || points.length <= size) return points;
|
||||
return points.slice(points.length - size);
|
||||
}
|
||||
|
||||
function describeSeries(points, metric) {
|
||||
if (!points.length) return "（无数据）";
|
||||
if (points.length === 1) return `仅 1 点 · 最新 ${fmtValue(metric, points[0][1])}`;
|
||||
const first = points[0], last = points[points.length - 1];
|
||||
const delta = last[1] - first[1];
|
||||
const sign = delta > 0 ? "+" : "";
|
||||
return `${points.length} 点 · 窗口内 ${sign}${fmtValue(metric, delta)} · ` +
|
||||
`最新 ${fmtValue(metric, last[1])} @ ${fmtInt(last[0])} 步`;
|
||||
}
|
||||
|
||||
function bindChartInteractions() {
|
||||
for (const id of ["chart-reward", "chart-metric"]) {
|
||||
const canvas = $(id);
|
||||
if (!canvas) continue;
|
||||
const overlay = document.createElement("canvas");
|
||||
overlay.className = "chart-overlay";
|
||||
overlay.style.width = "100%"; overlay.style.height = "100%";
|
||||
if (canvas.parentElement) {
|
||||
canvas.parentElement.style.position = "relative";
|
||||
canvas.parentElement.appendChild(overlay);
|
||||
}
|
||||
const move = (event) => {
|
||||
const show = $("chart-cursor").checked;
|
||||
const rect = canvas.getBoundingClientRect();
|
||||
const which = id === "chart-reward" ? "reward" : "metric";
|
||||
const layout = state.layouts && state.layouts[which];
|
||||
const ctx = overlay.getContext("2d");
|
||||
const dpr = window.devicePixelRatio || 1;
|
||||
if (overlay.width !== Math.round(rect.width * dpr) || overlay.height !== Math.round(rect.height * dpr)) {
|
||||
overlay.width = Math.round(rect.width * dpr);
|
||||
overlay.height = Math.round(rect.height * dpr);
|
||||
}
|
||||
ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
|
||||
ctx.clearRect(0, 0, rect.width, rect.height);
|
||||
if (!layout || !show || !isFinite(layout.bounds.x0)) return;
|
||||
window.Charts.drawCursor(overlay, layout, {
|
||||
mouse: { x: event.clientX - rect.left, y: event.clientY - rect.top },
|
||||
xFormat: (v) => `${fmtInt(v)}步`,
|
||||
});
|
||||
};
|
||||
canvas.addEventListener("mousemove", move);
|
||||
canvas.addEventListener("mouseleave", () => {
|
||||
const ctx = overlay.getContext("2d");
|
||||
ctx.clearRect(0, 0, overlay.width, overlay.height);
|
||||
});
|
||||
}
|
||||
$("chart-window").addEventListener("change", renderCharts);
|
||||
$("chart-cursor").addEventListener("change", renderCharts);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 日志
|
||||
function renderLog(job) {
|
||||
const lines = job.log_lines || [];
|
||||
const filter = ($("log-filter").value || "").trim().toLowerCase();
|
||||
const showJson = $("log-json").checked;
|
||||
const errorsOnly = $("log-errors-only").checked;
|
||||
const view = $("log-view");
|
||||
const signature = `${lines.length}|${job.log_total}|${filter}|${showJson}|${errorsOnly}|${lines[lines.length - 1] || ""}`;
|
||||
if (signature === state.lastLogSignature) return;
|
||||
state.lastLogSignature = signature;
|
||||
|
||||
const follow = $("log-follow").checked;
|
||||
const atBottom = view.scrollTop + view.clientHeight >= view.scrollHeight - 24;
|
||||
const keep = follow || atBottom;
|
||||
|
||||
const html = [];
|
||||
let shown = 0;
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const raw = lines[i];
|
||||
const lower = raw.toLowerCase();
|
||||
if (filter && !lower.includes(filter)) continue;
|
||||
if (errorsOnly && !/error|traceback|failed|exception|错误|失败/.test(lower)) continue;
|
||||
if (!showJson && raw.trim().startsWith("{") && raw.includes('"event"')) continue;
|
||||
shown++;
|
||||
const cls = classify(raw);
|
||||
html.push(`<span class="l ${cls}">${highlight(raw, filter)}</span>`);
|
||||
}
|
||||
view.innerHTML = html.length ? html.join("") : '<span class="l dim">（无匹配日志行）</span>';
|
||||
if (keep) view.scrollTop = view.scrollHeight;
|
||||
setText("log-shown", `显示 ${shown} / 共 ${fmtInt(job.log_total)} 行`);
|
||||
setText("log-meta", `stdout/stderr 合并捕获 · 服务端保留最近 ${fmtInt(job.process ? job.process.log_capacity : 4000)} 行`);
|
||||
}
|
||||
|
||||
function classify(line) {
|
||||
const text = line.toLowerCase();
|
||||
if (/error|traceback|exception|failed|错误|失败/.test(text)) return "err";
|
||||
if (/warn|deprecat|警告/.test(text)) return "warn";
|
||||
if (line.trim().startsWith("{")) return "json";
|
||||
if (/^\s*[-=]{3,}|已保存|success|complete|评估|metadata/.test(text)) return "meta";
|
||||
if (!line.trim()) return "dim";
|
||||
return "";
|
||||
}
|
||||
|
||||
function highlight(line, filter) {
|
||||
const escaped = escapeHtml(line);
|
||||
if (!filter) return escaped;
|
||||
try {
|
||||
return escaped.replace(new RegExp(`(${filter.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")})`, "gi"),
|
||||
"<mark>$1</mark>");
|
||||
} catch (err) { return escaped; }
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 逐回合
|
||||
function renderEpisodes(job) {
|
||||
const rows = job.monitor || [];
|
||||
const tbody = $("episode-table").querySelector("tbody");
|
||||
const tail = rows.slice(-200).reverse();
|
||||
tbody.innerHTML = tail.map((row, index) => {
|
||||
const seq = rows.length - index;
|
||||
return `<tr><td>${seq}</td><td>${fmtNumber(row.r)}</td>` +
|
||||
`<td>${fmtNumber(row.l, 0)}</td><td>${fmtNumber(row.t, 3)}</td></tr>`;
|
||||
}).join("") || '<tr><td colspan="4" class="hint">尚无逐回合记录（Monitor CSV 随训练写入）</td></tr>';
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 交互
|
||||
$("config-form").addEventListener("submit", async (event) => {
|
||||
event.preventDefault();
|
||||
if (state.saving) return;
|
||||
const err = $("form-error");
|
||||
err.textContent = "";
|
||||
const values = collectValues();
|
||||
const problems = validateLocally(values);
|
||||
if (problems.length) { err.textContent = problems[0]; return; }
|
||||
const payload = {};
|
||||
for (const [key, value] of Object.entries(values)) {
|
||||
if (value !== null) payload[key] = value;
|
||||
}
|
||||
if (payload.resume && !payload.model) { err.textContent = "续训需要填写模型输出路径"; return; }
|
||||
const res = await fetch("/api/train/start", {
|
||||
method: "POST", headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
const body = await res.json().catch(() => ({}));
|
||||
if (!res.ok) { err.textContent = body.error || `启动失败（HTTP ${res.status}）`; return; }
|
||||
state.preview = false;
|
||||
state.jobKey = null;
|
||||
state.metrics = [];
|
||||
state.metricCache = [];
|
||||
state.metricNext = 0;
|
||||
state.metricTotal = 0;
|
||||
await poll();
|
||||
});
|
||||
|
||||
$("btn-stop").addEventListener("click", async () => {
|
||||
if (!confirm("确认停止当前训练？\n将向独立进程组发送 SIGTERM，本轮模型可能未保存。")) return;
|
||||
const res = await fetch("/api/train/stop", {
|
||||
method: "POST", headers: { "Content-Type": "application/json" }, body: "{}",
|
||||
});
|
||||
const body = await res.json().catch(() => ({}));
|
||||
if (!res.ok) showBanner("bad", `停止失败：${escapeHtml(body.error || res.status)}`);
|
||||
await poll();
|
||||
});
|
||||
|
||||
$("btn-refresh").addEventListener("click", async () => {
|
||||
state.preview = false;
|
||||
showBanner("", "");
|
||||
await poll();
|
||||
});
|
||||
|
||||
$("btn-preview").addEventListener("click", async () => {
|
||||
try {
|
||||
const scenario = $("preview-scenario") ? $("preview-scenario").value : "rising";
|
||||
const res = await fetch(`/api/train/state?preview=1&scenario=${encodeURIComponent(scenario)}&log_tail=400`);
|
||||
const payload = await res.json();
|
||||
if (!payload.job) throw new Error(payload.error || "服务端未返回示例快照");
|
||||
state.preview = true;
|
||||
state.jobKey = null;
|
||||
state.metrics = [];
|
||||
state.metricCache = [];
|
||||
state.metricNext = 0;
|
||||
state.metricTotal = 0;
|
||||
render(payload.job);
|
||||
} catch (err) {
|
||||
showBanner("bad", `离线预览不可用：${escapeHtml(err.message)}`);
|
||||
}
|
||||
});
|
||||
|
||||
$("btn-reset").addEventListener("click", () => {
|
||||
applyValues({});
|
||||
$("form-error").textContent = "";
|
||||
});
|
||||
|
||||
for (const id of ["log-filter", "log-json", "log-errors-only", "log-tail"]) {
|
||||
$(id).addEventListener("input", () => {
|
||||
state.lastLogSignature = "";
|
||||
poll();
|
||||
});
|
||||
}
|
||||
|
||||
boot();
|
||||
@@ -0,0 +1,255 @@
|
||||
/* 训练前端图表：原生 canvas 折线图，无外部依赖（离线可用，与评测前端同风格）。
|
||||
只做三件事：坐标轴自适应、多序列折线、悬停读数；不引入缩放/图例等交互，
|
||||
因为训练曲线每 2 秒整体重绘一次，重量级图表库在这里只会增加风险。
|
||||
|
||||
对外接口（挂到 globalThis.Charts）：
|
||||
Charts.fit(canvas) 按 DPR 调整画布像素
|
||||
Charts.line(canvas, {series, refs, ...}) 折线图
|
||||
Charts.sparkline(canvas, values, opts) 指标卡片内的小趋势线
|
||||
数据约定：series = [{ points: [[x, y], ...], color, width, fill }] */
|
||||
(function (global) {
|
||||
"use strict";
|
||||
|
||||
var AXIS = "#2a3550", GRID = "rgba(42,53,80,.55)", DIM = "#93a1bd", TEXT = "#e6ecf8";
|
||||
var FONT = '11px ui-monospace, SFMono-Regular, Menlo, monospace';
|
||||
|
||||
function fit(canvas) {
|
||||
var dpr = global.devicePixelRatio || 1;
|
||||
var rect = canvas.getBoundingClientRect ? canvas.getBoundingClientRect() : null;
|
||||
var cssWidth = (rect && rect.width) || canvas.clientWidth || canvas.width || 640;
|
||||
var cssHeight = (rect && rect.height) || canvas.clientHeight || canvas.height || 200;
|
||||
var w = Math.max(160, Math.round(cssWidth * dpr));
|
||||
var h = Math.max(80, Math.round(cssHeight * dpr));
|
||||
if (canvas.width !== w || canvas.height !== h) { canvas.width = w; canvas.height = h; }
|
||||
var ctx = canvas.getContext("2d");
|
||||
ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
|
||||
return { ctx: ctx, w: w / dpr, h: h / dpr };
|
||||
}
|
||||
|
||||
/* 第 0 层：数值格式（科学计数法只用于真正小而大的量级） */
|
||||
function fmt(value) {
|
||||
if (value === null || value === undefined || Number.isNaN(value)) return "—";
|
||||
var abs = Math.abs(value);
|
||||
if (abs !== 0 && (abs < 1e-3 || abs >= 1e5)) return value.toExponential(2);
|
||||
if (abs >= 100) return value.toFixed(1);
|
||||
if (abs >= 1) return value.toFixed(2);
|
||||
if (abs >= 0.01) return value.toFixed(3);
|
||||
return value.toFixed(4);
|
||||
}
|
||||
|
||||
function niceTicks(min, max, count) {
|
||||
if (!isFinite(min) || !isFinite(max)) return [0, 1];
|
||||
if (min === max) { var pad = Math.abs(min) > 1e-9 ? Math.abs(min) * 0.1 : 0.5;
|
||||
min -= pad; max += pad; }
|
||||
var span = max - min;
|
||||
var raw = span / Math.max(1, count);
|
||||
var mag = Math.pow(10, Math.floor(Math.log10(raw)));
|
||||
var norm = raw / mag;
|
||||
var step = (norm <= 1 ? 1 : norm <= 2 ? 2 : norm <= 5 ? 5 : 10) * mag;
|
||||
var start = Math.ceil(min / step) * step;
|
||||
var ticks = [];
|
||||
for (var v = start; v <= max + step * 1e-6 && ticks.length < 40; v += step) {
|
||||
ticks.push(Math.abs(v) < step * 1e-9 ? 0 : v);
|
||||
}
|
||||
if (!ticks.length) ticks = [min, max];
|
||||
return ticks;
|
||||
}
|
||||
|
||||
function bounds(series) {
|
||||
var xs = [], ys = [];
|
||||
for (var i = 0; i < series.length; i++) {
|
||||
var pts = series[i].points || [];
|
||||
for (var j = 0; j < pts.length; j++) {
|
||||
var p = pts[j];
|
||||
if (!p || !isFinite(p[0]) || !isFinite(p[1])) continue;
|
||||
xs.push(p[0]); ys.push(p[1]);
|
||||
}
|
||||
}
|
||||
if (!xs.length) return null;
|
||||
var x0 = Math.min.apply(null, xs), x1 = Math.max.apply(null, xs);
|
||||
var y0 = Math.min.apply(null, ys), y1 = Math.max.apply(null, ys);
|
||||
if (x0 === x1) { x0 -= 0.5; x1 += 0.5; }
|
||||
if (y0 === y1) { var pad = Math.abs(y0) > 1e-9 ? Math.abs(y0) * 0.08 : 0.5;
|
||||
y0 -= pad; y1 += pad; }
|
||||
return { x0: x0, x1: x1, y0: y0, y1: y1 };
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------ 折线图 */
|
||||
function line(canvas, options) {
|
||||
var opts = options || {};
|
||||
var axes = fit(canvas), ctx = axes.ctx, W = axes.w, H = axes.h;
|
||||
var padL = opts.padLeft || 62, padR = 14, padT = 10, padB = 24;
|
||||
ctx.clearRect(0, 0, W, H);
|
||||
|
||||
var series = (opts.series || []).filter(function (s) {
|
||||
return s && s.points && s.points.length;
|
||||
});
|
||||
var box = bounds(series);
|
||||
if (!box) {
|
||||
ctx.fillStyle = DIM; ctx.font = FONT;
|
||||
ctx.fillText(opts.emptyText || "等待数据……", padL, H / 2);
|
||||
return null;
|
||||
}
|
||||
var yTicks = niceTicks(box.y0, box.y1, 4);
|
||||
var yMin = Math.min(box.y0, yTicks[0]), yMax = Math.max(box.y1, yTicks[yTicks.length - 1]);
|
||||
var plotW = Math.max(10, W - padL - padR), plotH = Math.max(10, H - padT - padB);
|
||||
var sx = function (x) { return padL + (x - box.x0) / (box.x1 - box.x0) * plotW; };
|
||||
var sy = function (y) { return padT + (1 - (y - yMin) / (yMax - yMin)) * plotH; };
|
||||
|
||||
// 网格与刻度
|
||||
ctx.font = FONT; ctx.lineWidth = 1;
|
||||
for (var i = 0; i < yTicks.length; i++) {
|
||||
var y = sy(yTicks[i]);
|
||||
ctx.strokeStyle = GRID; ctx.beginPath();
|
||||
ctx.moveTo(padL, y + 0.5); ctx.lineTo(padL + plotW, y + 0.5); ctx.stroke();
|
||||
ctx.fillStyle = DIM; ctx.textAlign = "right";
|
||||
ctx.fillText(fmt(yTicks[i]), padL - 6, y + 3.5);
|
||||
}
|
||||
ctx.textAlign = "center";
|
||||
for (var t = 0; t <= 4; t++) {
|
||||
var xv = box.x0 + (box.x1 - box.x0) * (t / 4);
|
||||
var x = sx(xv);
|
||||
ctx.fillStyle = DIM;
|
||||
ctx.fillText(opts.xFormat ? opts.xFormat(xv) : fmt(xv), x, H - 7);
|
||||
}
|
||||
ctx.textAlign = "left";
|
||||
ctx.strokeStyle = AXIS; ctx.beginPath();
|
||||
ctx.moveTo(padL + 0.5, padT); ctx.lineTo(padL + 0.5, padT + plotH);
|
||||
ctx.lineTo(padL + plotW, padT + plotH); ctx.stroke();
|
||||
|
||||
// 参考线（例如 KL 的 0.05 阈值、解释方差的 0）
|
||||
var refs = opts.refs || [];
|
||||
for (var r = 0; r < refs.length; r++) {
|
||||
var ref = refs[r];
|
||||
if (ref.value === undefined || ref.value < yMin || ref.value > yMax) continue;
|
||||
var ry = sy(ref.value);
|
||||
ctx.save();
|
||||
ctx.setLineDash([4, 4]); ctx.strokeStyle = ref.color || "#f59e0b";
|
||||
ctx.beginPath(); ctx.moveTo(padL, ry); ctx.lineTo(padL + plotW, ry); ctx.stroke();
|
||||
ctx.restore();
|
||||
if (ref.label) {
|
||||
ctx.fillStyle = ref.color || "#f59e0b"; ctx.font = FONT;
|
||||
ctx.fillText(ref.label, padL + 6, ry - 4);
|
||||
}
|
||||
}
|
||||
|
||||
// 序列
|
||||
for (var s = 0; s < series.length; s++) {
|
||||
var item = series[s];
|
||||
var pts = item.points;
|
||||
ctx.strokeStyle = item.color || "#4f8cff";
|
||||
ctx.lineWidth = item.width || 1.6;
|
||||
ctx.lineJoin = "round";
|
||||
ctx.beginPath();
|
||||
for (var k = 0; k < pts.length; k++) {
|
||||
var px = sx(pts[k][0]), py = sy(pts[k][1]);
|
||||
if (k === 0) ctx.moveTo(px, py); else ctx.lineTo(px, py);
|
||||
}
|
||||
ctx.stroke();
|
||||
if (item.fill && pts.length > 1) {
|
||||
ctx.save();
|
||||
ctx.globalAlpha = 0.12; ctx.fillStyle = item.color || "#4f8cff";
|
||||
ctx.lineTo(sx(pts[pts.length - 1][0]), padT + plotH);
|
||||
ctx.lineTo(sx(pts[0][0]), padT + plotH);
|
||||
ctx.closePath(); ctx.fill(); ctx.restore();
|
||||
}
|
||||
// 末点高亮：一眼看到最新值
|
||||
var last = pts[pts.length - 1];
|
||||
ctx.fillStyle = item.color || "#4f8cff";
|
||||
ctx.beginPath(); ctx.arc(sx(last[0]), sy(last[1]), 2.6, 0, Math.PI * 2); ctx.fill();
|
||||
}
|
||||
|
||||
return { bounds: box, sx: sx, sy: sy, padL: padL, padT: padT,
|
||||
plotW: plotW, plotH: plotH, yMin: yMin, yMax: yMax, series: series };
|
||||
}
|
||||
|
||||
/* 悬停读数：把鼠标 x 映射到最接近的数据点，返回一次绘制的覆盖层信息 */
|
||||
function drawCursor(canvas, layout, options) {
|
||||
var opts = options || {};
|
||||
if (!layout || !layout.series || !layout.series.length) return;
|
||||
var axes = fit(canvas), ctx = axes.ctx;
|
||||
var mouse = opts.mouse;
|
||||
if (!mouse || mouse.x < layout.padL || mouse.x > layout.padL + layout.plotW) return;
|
||||
var target = layout.bounds.x0 +
|
||||
(mouse.x - layout.padL) / layout.plotW * (layout.bounds.x1 - layout.bounds.x0);
|
||||
var lines = [], snapX = target;
|
||||
for (var s = 0; s < layout.series.length; s++) {
|
||||
var pts = layout.series[s].points;
|
||||
if (!pts.length) continue;
|
||||
// 数据已按 x 递增，二分定位最近点
|
||||
var lo = 0, hi = pts.length - 1, best = pts[hi];
|
||||
while (lo <= hi) {
|
||||
var mid = (lo + hi) >> 1;
|
||||
if (pts[mid][0] === target) { best = pts[mid]; break; }
|
||||
if (pts[mid][0] < target) { best = pts[mid]; lo = mid + 1; }
|
||||
else { hi = mid - 1; }
|
||||
}
|
||||
var next = pts[Math.min(pts.length - 1, lo)];
|
||||
if (next && Math.abs(next[0] - target) < Math.abs(best[0] - target)) best = next;
|
||||
snapX = best[0];
|
||||
lines.push({ label: layout.series[s].name || "", color: layout.series[s].color,
|
||||
value: best[1] });
|
||||
}
|
||||
var cx = layout.sx(snapX);
|
||||
ctx.save();
|
||||
ctx.setLineDash([3, 3]); ctx.strokeStyle = "rgba(230,236,248,.35)";
|
||||
ctx.beginPath(); ctx.moveTo(cx, layout.padT);
|
||||
ctx.lineTo(cx, layout.padT + layout.plotH); ctx.stroke();
|
||||
ctx.restore();
|
||||
for (var i = 0; i < lines.length; i++) {
|
||||
var item = lines[i];
|
||||
var cy = layout.sy(item.value);
|
||||
ctx.fillStyle = item.color || "#4f8cff";
|
||||
ctx.beginPath(); ctx.arc(cx, cy, 3.2, 0, Math.PI * 2); ctx.fill();
|
||||
ctx.strokeStyle = "rgba(11,15,26,.9)"; ctx.lineWidth = 1.4; ctx.stroke();
|
||||
}
|
||||
var text = (opts.xFormat ? opts.xFormat(snapX) : fmt(snapX)) + " · " +
|
||||
lines.map(function (item) { return (item.label ? item.label + " " : "") + fmt(item.value); })
|
||||
.join(" ");
|
||||
ctx.font = FONT;
|
||||
var tw = ctx.measureText(text).width + 14;
|
||||
var bx = Math.min(cx + 8, layout.padL + layout.plotW - tw);
|
||||
bx = Math.max(layout.padL + 2, bx);
|
||||
var by = layout.padT + 4;
|
||||
ctx.fillStyle = "rgba(23,30,46,.94)"; ctx.strokeStyle = AXIS;
|
||||
ctx.beginPath();
|
||||
if (ctx.roundRect) { ctx.roundRect(bx, by, tw, 20, 6); ctx.fill(); ctx.stroke(); }
|
||||
else { ctx.fillRect(bx, by, tw, 20); ctx.strokeRect(bx, by, tw, 20); }
|
||||
ctx.fillStyle = TEXT; ctx.textAlign = "left";
|
||||
ctx.fillText(text, bx + 7, by + 14);
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------ 迷你趋势线 */
|
||||
function sparkline(canvas, values, options) {
|
||||
var opts = options || {};
|
||||
var axes = fit(canvas), ctx = axes.ctx, W = axes.w, H = axes.h;
|
||||
ctx.clearRect(0, 0, W, H);
|
||||
var clean = (values || []).filter(function (v) { return isFinite(v); });
|
||||
if (clean.length < 2) {
|
||||
ctx.strokeStyle = GRID; ctx.beginPath();
|
||||
ctx.moveTo(0, H - 1.5); ctx.lineTo(W, H - 1.5); ctx.stroke();
|
||||
return;
|
||||
}
|
||||
var lo = Math.min.apply(null, clean), hi = Math.max.apply(null, clean);
|
||||
if (lo === hi) { lo -= 0.5; hi += 0.5; }
|
||||
var pad = 2;
|
||||
var X = function (i) { return (i / (clean.length - 1)) * W; };
|
||||
var Y = function (v) { return H - pad - (v - lo) / (hi - lo) * (H - pad * 2); };
|
||||
ctx.strokeStyle = opts.color || "#4f8cff"; ctx.lineWidth = 1.4;
|
||||
ctx.beginPath();
|
||||
for (var i = 0; i < clean.length; i++) {
|
||||
if (i === 0) ctx.moveTo(X(i), Y(clean[i])); else ctx.lineTo(X(i), Y(clean[i]));
|
||||
}
|
||||
ctx.stroke();
|
||||
if (opts.fill !== false) {
|
||||
ctx.save(); ctx.globalAlpha = 0.14; ctx.fillStyle = opts.color || "#4f8cff";
|
||||
ctx.lineTo(W, H); ctx.lineTo(0, H); ctx.closePath(); ctx.fill(); ctx.restore();
|
||||
}
|
||||
var lastX = X(clean.length - 1), lastY = Y(clean[clean.length - 1]);
|
||||
ctx.fillStyle = opts.color || "#4f8cff";
|
||||
ctx.beginPath(); ctx.arc(lastX - 1.5, lastY, 2.2, 0, Math.PI * 2); ctx.fill();
|
||||
}
|
||||
|
||||
global.Charts = { fit: fit, line: line, drawCursor: drawCursor,
|
||||
sparkline: sparkline, fmt: fmt };
|
||||
})(typeof globalThis !== "undefined" ? globalThis : this);
|
||||
@@ -0,0 +1,174 @@
|
||||
<!doctype html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>模型训练前端 · CUMCM 2026 B HRL</title>
|
||||
<link rel="stylesheet" href="/style.css">
|
||||
</head>
|
||||
<body>
|
||||
<header class="topbar">
|
||||
<div class="brand">
|
||||
<h1>模型训练前端 <span class="sub">HRL 高层 MaskablePPO</span></h1>
|
||||
<p class="tagline" id="job-line">尚未启动训练任务</p>
|
||||
</div>
|
||||
<div class="topstatus">
|
||||
<span class="chip gray" id="chip-process">进程<b>未启动</b></span>
|
||||
<span class="chip gray" id="chip-progress">进度<b>—</b></span>
|
||||
<span class="chip gray" id="chip-elapsed">墙钟<b>—</b></span>
|
||||
<span class="chip gray" id="chip-device">设备<b>—</b></span>
|
||||
<span class="chip gray" id="chip-conn">连接<b>探测中</b></span>
|
||||
</div>
|
||||
<div class="topactions">
|
||||
<button id="btn-refresh" type="button" title="立即拉取一次快照">刷新</button>
|
||||
<label class="inline" title="离线预览：读取示例快照，用于无训练进程时检查界面">
|
||||
预览
|
||||
<select id="preview-scenario">
|
||||
<option value="rising">上升</option>
|
||||
<option value="plateau">平台期</option>
|
||||
<option value="kl-spike">KL 过猛</option>
|
||||
</select>
|
||||
</label>
|
||||
<button id="btn-preview" type="button" title="渲染示例快照，不启动任何训练进程">离线预览</button>
|
||||
<button id="btn-stop" class="danger" type="button" disabled>停止训练</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<div id="banner" class="banner hidden"></div>
|
||||
|
||||
<!-- =============================================== 1. 执行进度与进程 -->
|
||||
<section class="panel" id="panel-process">
|
||||
<div class="panel-head">
|
||||
<h2>执行进度与进程</h2>
|
||||
<span class="hint" id="process-hint">启动任务后显示子进程与步进进度</span>
|
||||
</div>
|
||||
|
||||
<div class="progress-row">
|
||||
<div class="progress-main">
|
||||
<div class="progress-labels">
|
||||
<span><b id="steps-done">0</b> / <span id="steps-total">0</span> 环境步</span>
|
||||
<span class="hint" id="progress-percent">0.0%</span>
|
||||
</div>
|
||||
<div class="bar"><div class="bar-fill" id="bar-fill"></div></div>
|
||||
<div class="progress-foot">
|
||||
<span class="hint">剩余估计 <b id="eta">—</b></span>
|
||||
<span class="hint">预计结束 <b id="eta-clock">—</b></span>
|
||||
<span class="hint">进度点 <b id="metric-total">0</b></span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="stat-grid">
|
||||
<div class="stat"><span class="k">子进程 PID</span><b class="v" id="stat-pid">—</b></div>
|
||||
<div class="stat"><span class="k">进程组 / 会话</span><b class="v" id="stat-pgid">独立</b></div>
|
||||
<div class="stat"><span class="k">吞吐 FPS</span><b class="v" id="stat-fps">—</b></div>
|
||||
<div class="stat"><span class="k">已用墙钟</span><b class="v" id="stat-elapsed">—</b></div>
|
||||
<div class="stat"><span class="k">退出码</span><b class="v" id="stat-rc">—</b></div>
|
||||
<div class="stat"><span class="k">日志行数</span><b class="v" id="stat-logs">0</b></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="timeline" id="timeline"></div>
|
||||
<p class="hint" id="command-preview"></p>
|
||||
</section>
|
||||
|
||||
<!-- =============================================== 2. 训练配置 -->
|
||||
<section class="panel" id="panel-config">
|
||||
<div class="panel-head">
|
||||
<h2>训练配置</h2>
|
||||
<span class="hint">字段与后端 <code>TrainConfig</code> 同源；留空字段取默认值</span>
|
||||
</div>
|
||||
<div class="presets" id="presets"></div>
|
||||
<form id="config-form" autocomplete="off">
|
||||
<div class="form-groups" id="form-groups"></div>
|
||||
<div class="form-foot">
|
||||
<button id="btn-start" class="primary" type="submit">开始训练</button>
|
||||
<button id="btn-reset" type="button">恢复默认</button>
|
||||
<span class="form-error" id="form-error"></span>
|
||||
</div>
|
||||
</form>
|
||||
</section>
|
||||
|
||||
<!-- =============================================== 3. 参数收敛 -->
|
||||
<section class="panel" id="panel-convergence">
|
||||
<div class="panel-head">
|
||||
<h2>参数收敛状态</h2>
|
||||
<span class="hint">判据为最近窗口的斜率与波动，仅供人工复核</span>
|
||||
</div>
|
||||
<div class="converge-head">
|
||||
<span class="chip gray" id="chip-converge">判读<b>等待数据</b></span>
|
||||
<ul class="notes" id="converge-notes"></ul>
|
||||
</div>
|
||||
<div class="metric-grid" id="metric-grid"></div>
|
||||
|
||||
<div class="chart-toolbar">
|
||||
<div class="chart-tabs" id="chart-tabs"></div>
|
||||
<label class="inline">最近点数
|
||||
<select id="chart-window">
|
||||
<option value="0">全部</option>
|
||||
<option value="40">40</option>
|
||||
<option value="120" selected>120</option>
|
||||
<option value="400">400</option>
|
||||
</select>
|
||||
</label>
|
||||
<label class="inline"><input type="checkbox" id="chart-cursor" checked> 悬停读数</label>
|
||||
</div>
|
||||
<div class="chart-stack">
|
||||
<figure class="chart-figure">
|
||||
<figcaption><b id="chart-title">平均回合奖励</b><span class="hint" id="chart-sub"></span></figcaption>
|
||||
<canvas id="chart-reward" height="240"></canvas>
|
||||
</figure>
|
||||
<figure class="chart-figure">
|
||||
<figcaption><b id="chart2-title">近似 KL</b><span class="hint" id="chart2-sub"></span></figcaption>
|
||||
<canvas id="chart-metric" height="180"></canvas>
|
||||
</figure>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- =============================================== 4. 实时运行日志 -->
|
||||
<section class="panel" id="panel-log">
|
||||
<div class="panel-head">
|
||||
<h2>实时运行日志</h2>
|
||||
<span class="hint" id="log-meta">stdout/stderr 合并捕获</span>
|
||||
</div>
|
||||
<div class="log-toolbar">
|
||||
<input id="log-filter" type="search" placeholder="过滤关键字，如 rollout / progress / error">
|
||||
<label class="inline"><input type="checkbox" id="log-json" checked> 显示进度 JSON</label>
|
||||
<label class="inline"><input type="checkbox" id="log-errors-only"> 只看错误</label>
|
||||
<label class="inline"><input type="checkbox" id="log-follow" checked> 自动跟随到底部</label>
|
||||
<label class="inline">尾部行数
|
||||
<select id="log-tail">
|
||||
<option value="200">200</option>
|
||||
<option value="400" selected>400</option>
|
||||
<option value="1000">1000</option>
|
||||
<option value="2000">2000</option>
|
||||
</select>
|
||||
</label>
|
||||
<span class="hint" id="log-shown"></span>
|
||||
</div>
|
||||
<pre class="log-view" id="log-view">（暂无输出）</pre>
|
||||
</section>
|
||||
|
||||
<!-- =============================================== 5. 逐回合奖励 -->
|
||||
<section class="panel" id="panel-episodes">
|
||||
<div class="panel-head">
|
||||
<h2>逐回合记录</h2>
|
||||
<span class="hint">来自训练脚本写入的 Monitor CSV（每回合奖励/长度/时刻）</span>
|
||||
</div>
|
||||
<div class="table-wrap">
|
||||
<table id="episode-table">
|
||||
<thead><tr><th>#</th><th>奖励 r</th><th>长度 l</th><th>时刻 t (s)</th></tr></thead>
|
||||
<tbody></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</section>
|
||||
</main>
|
||||
|
||||
<footer>
|
||||
本地回环训练前端：与评测前端共用「白名单静态资源 + JSON 轮询 API」模式；
|
||||
命令由服务端以参数列表构造（不经过 shell），停止时向独立进程组发送 SIGTERM。
|
||||
</footer>
|
||||
|
||||
<script src="/charts.js"></script>
|
||||
<script src="/app.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,221 @@
|
||||
/* 训练前端样式：沿用评测前端（referee/static/style.css）的暗色变量与圆角面板
|
||||
语言，保证两个界面像同一套系统的两个页面；本页信息密度更高：
|
||||
进度条、进程时间线、参数卡片网格、曲线与日志终端。 */
|
||||
|
||||
:root {
|
||||
--bg: #0f1420; --panel: #171e2e; --panel2: #1d2740; --line: #2a3550;
|
||||
--text: #e6ecf8; --dim: #93a1bd; --accent: #4f8cff; --ok: #34d399;
|
||||
--warn: #f59e0b; --bad: #f87171; --cyan: #38bdf8; --gray: #6b7280;
|
||||
--mono: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
body {
|
||||
margin: 0; background: var(--bg); color: var(--text);
|
||||
font: 14px/1.5 "PingFang SC", "Microsoft YaHei", system-ui, sans-serif;
|
||||
}
|
||||
code, .mono { background: var(--panel2); padding: 1px 6px; border-radius: 4px; color: var(--cyan); font-family: var(--mono); font-size: 12px; }
|
||||
main { max-width: 1240px; margin: 0 auto; padding: 16px 22px 48px; }
|
||||
footer { color: var(--dim); font-size: 12px; text-align: center; padding: 14px 22px; border-top: 1px solid var(--line); }
|
||||
.hidden { display: none !important; }
|
||||
.hint { color: var(--dim); font-size: 12px; }
|
||||
h2 { font-size: 15px; margin: 0; }
|
||||
|
||||
/* ------------------------------------------------------------ 顶栏 */
|
||||
.topbar {
|
||||
position: sticky; top: 0; z-index: 20;
|
||||
display: flex; align-items: center; gap: 18px; flex-wrap: wrap;
|
||||
padding: 12px 22px; border-bottom: 1px solid var(--line);
|
||||
background: rgba(23,30,46,.96); backdrop-filter: blur(6px);
|
||||
}
|
||||
.brand h1 { font-size: 17px; margin: 0; }
|
||||
.brand .sub { font-size: 12px; color: var(--dim); font-weight: normal; margin-left: 8px; }
|
||||
.brand .tagline { margin: 3px 0 0; color: var(--dim); font-size: 12px; font-family: var(--mono); }
|
||||
.topstatus { display: flex; gap: 8px; flex-wrap: wrap; margin-left: auto; }
|
||||
.topactions { display: flex; gap: 8px; }
|
||||
|
||||
.chip {
|
||||
background: var(--panel2); border: 1px solid var(--line); border-radius: 999px;
|
||||
padding: 3px 12px; font-size: 12px; color: var(--dim); white-space: nowrap;
|
||||
}
|
||||
.chip b { color: var(--text); font-weight: 600; margin-left: 4px; }
|
||||
.chip.gray b { color: var(--gray); }
|
||||
.chip.ok b { color: var(--ok); }
|
||||
.chip.warn b { color: var(--warn); }
|
||||
.chip.bad b { color: var(--bad); }
|
||||
.chip.cyan b { color: var(--cyan); }
|
||||
|
||||
button, .button {
|
||||
background: var(--panel2); color: var(--text); border: 1px solid var(--line);
|
||||
border-radius: 6px; padding: 7px 14px; font-size: 13px; cursor: pointer;
|
||||
}
|
||||
button:hover:enabled, .button:hover { border-color: var(--accent); }
|
||||
button:disabled { opacity: .45; cursor: not-allowed; }
|
||||
button.primary { background: var(--accent); border-color: var(--accent); color: #fff; }
|
||||
button.primary:hover:enabled { filter: brightness(1.08); }
|
||||
button.danger { background: transparent; border-color: var(--bad); color: var(--bad); }
|
||||
button.danger:hover:enabled { background: rgba(248,113,113,.12); }
|
||||
|
||||
.banner {
|
||||
border: 1px solid var(--bad); border-left: 4px solid var(--bad);
|
||||
background: rgba(248,113,113,.08); border-radius: 8px;
|
||||
padding: 10px 14px; margin: 14px 0 0; font-size: 13px;
|
||||
}
|
||||
.banner.ok { border-color: var(--ok); border-left-color: var(--ok); background: rgba(52,211,153,.08); }
|
||||
.banner.warn { border-color: var(--warn); border-left-color: var(--warn); background: rgba(245,158,11,.08); }
|
||||
.banner b { font-family: var(--mono); }
|
||||
|
||||
/* ------------------------------------------------------------ 面板 */
|
||||
.panel {
|
||||
border: 1px solid var(--line); border-radius: 12px; background: var(--panel);
|
||||
padding: 14px 16px; margin-top: 16px;
|
||||
}
|
||||
.panel-head { display: flex; align-items: baseline; gap: 12px; margin-bottom: 12px; flex-wrap: wrap; }
|
||||
.panel-head h2 { color: var(--text); }
|
||||
.panel-head h2::before {
|
||||
content: ""; display: inline-block; width: 3px; height: 13px; margin-right: 8px;
|
||||
background: var(--accent); border-radius: 2px; vertical-align: -1px;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------ 进度 */
|
||||
.progress-row { display: flex; gap: 20px; align-items: stretch; flex-wrap: wrap; }
|
||||
.progress-main { flex: 1 1 380px; min-width: 320px; }
|
||||
.progress-labels { display: flex; justify-content: space-between; font-size: 13px; margin-bottom: 6px; }
|
||||
.progress-labels b { font-family: var(--mono); font-size: 15px; }
|
||||
.bar { height: 12px; border-radius: 999px; background: #0b0f1a; border: 1px solid var(--line); overflow: hidden; }
|
||||
.bar-fill {
|
||||
height: 100%; width: 0%; border-radius: 999px;
|
||||
background: linear-gradient(90deg, #2f6fe0, var(--accent) 60%, var(--cyan));
|
||||
transition: width .3s ease;
|
||||
}
|
||||
.bar-fill.warn { background: linear-gradient(90deg, #b45309, var(--warn)); }
|
||||
.bar-fill.bad { background: linear-gradient(90deg, #b91c1c, var(--bad)); }
|
||||
.bar-fill.ok { background: linear-gradient(90deg, #059669, var(--ok)); }
|
||||
.progress-foot { display: flex; gap: 18px; margin-top: 7px; flex-wrap: wrap; }
|
||||
|
||||
.stat-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(132px, 1fr)); gap: 8px; flex: 1 1 420px; }
|
||||
.stat {
|
||||
border: 1px solid var(--line); border-radius: 8px; background: var(--panel2);
|
||||
padding: 7px 10px; display: flex; flex-direction: column; gap: 2px;
|
||||
}
|
||||
.stat .k { color: var(--dim); font-size: 11px; }
|
||||
.stat .v { font-family: var(--mono); font-size: 15px; }
|
||||
.stat .v.ok { color: var(--ok); }
|
||||
.stat .v.warn { color: var(--warn); }
|
||||
.stat .v.bad { color: var(--bad); }
|
||||
|
||||
.timeline { display: flex; align-items: center; gap: 0; margin-top: 16px; flex-wrap: wrap; }
|
||||
.node { display: flex; align-items: center; gap: 8px; }
|
||||
.node .dot {
|
||||
width: 12px; height: 12px; border-radius: 50%; border: 2px solid var(--line);
|
||||
background: var(--panel2); flex: none;
|
||||
}
|
||||
.node .txt { font-size: 12px; color: var(--dim); }
|
||||
.node.done .dot { border-color: var(--ok); background: var(--ok); }
|
||||
.node.done .txt { color: var(--text); }
|
||||
.node.active .dot { border-color: var(--accent); background: var(--accent); box-shadow: 0 0 0 4px rgba(79,140,255,.18); }
|
||||
.node.active .txt { color: var(--accent); font-weight: 600; }
|
||||
.node.failed .dot { border-color: var(--bad); background: var(--bad); }
|
||||
.node.failed .txt { color: var(--bad); }
|
||||
.link { width: 34px; height: 2px; background: var(--line); flex: none; margin: 0 8px; }
|
||||
.link.done { background: var(--ok); }
|
||||
#command-preview { margin: 12px 0 0; font-family: var(--mono); word-break: break-all; line-height: 1.7; }
|
||||
|
||||
/* ------------------------------------------------------------ 表单 */
|
||||
.presets { display: flex; flex-wrap: wrap; gap: 8px; margin-bottom: 14px; }
|
||||
.preset {
|
||||
border: 1px solid var(--line); border-radius: 8px; background: var(--panel2);
|
||||
padding: 7px 12px; font-size: 12px; cursor: pointer; color: var(--text);
|
||||
}
|
||||
.preset:hover { border-color: var(--accent); }
|
||||
.preset .note { display: block; color: var(--dim); font-size: 11px; margin-top: 2px; }
|
||||
.form-groups { display: grid; grid-template-columns: repeat(auto-fit, minmax(268px, 1fr)); gap: 14px; }
|
||||
fieldset.group { border: 1px solid var(--line); border-radius: 10px; padding: 10px 12px 12px; margin: 0; background: #141b29; }
|
||||
fieldset.group legend { color: var(--dim); font-size: 12px; padding: 0 6px; }
|
||||
fieldset.group .ghint { color: var(--dim); font-size: 11px; margin: 0 0 8px; }
|
||||
.field { display: flex; flex-direction: column; gap: 4px; margin-bottom: 10px; }
|
||||
.field .flabel { font-size: 12px; color: var(--dim); display: flex; gap: 6px; align-items: baseline; }
|
||||
.field .flabel .unit { color: var(--gray); font-size: 11px; }
|
||||
.field input[type="text"], .field input[type="number"], .field select {
|
||||
background: var(--panel2); border: 1px solid var(--line); color: var(--text);
|
||||
border-radius: 6px; padding: 6px 9px; font-size: 13px; width: 100%;
|
||||
font-family: var(--mono);
|
||||
}
|
||||
.field input:focus, .field select:focus, .log-toolbar input:focus { outline: none; border-color: var(--accent); }
|
||||
.field .fhelp { font-size: 11px; color: var(--gray); }
|
||||
.field.check { flex-direction: row; align-items: center; gap: 8px; }
|
||||
.field.check input { width: auto; }
|
||||
.field.dirty input, .field.dirty select { border-color: rgba(79,140,255,.65); }
|
||||
.field.invalid input, .field.invalid select { border-color: var(--bad); }
|
||||
.form-foot { display: flex; align-items: center; gap: 12px; margin-top: 14px; flex-wrap: wrap; }
|
||||
.form-error { color: var(--bad); font-size: 12px; }
|
||||
|
||||
/* ------------------------------------------------------------ 收敛 */
|
||||
.converge-head { display: flex; gap: 14px; align-items: flex-start; flex-wrap: wrap; margin-bottom: 12px; }
|
||||
.notes { margin: 0; padding-left: 18px; color: var(--dim); font-size: 12px; max-width: 760px; }
|
||||
.notes li { margin: 2px 0; }
|
||||
.metric-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(186px, 1fr)); gap: 10px; }
|
||||
.metric-card {
|
||||
border: 1px solid var(--line); border-radius: 10px; background: var(--panel2);
|
||||
padding: 9px 11px; display: flex; flex-direction: column; gap: 5px;
|
||||
}
|
||||
.metric-card .mhead { display: flex; justify-content: space-between; align-items: baseline; gap: 8px; }
|
||||
.metric-card .mname { font-size: 12px; color: var(--dim); }
|
||||
.metric-card .mvalue { font-family: var(--mono); font-size: 17px; }
|
||||
.metric-card .mfoot { display: flex; justify-content: space-between; font-size: 11px; color: var(--gray); gap: 6px; }
|
||||
.metric-card canvas { width: 100%; height: 40px; display: block; }
|
||||
.trend.up { color: var(--ok); } .trend.down { color: var(--warn); }
|
||||
.trend.flat { color: var(--dim); } .trend.none { color: var(--gray); }
|
||||
.metric-card.alert { border-color: rgba(248,113,113,.55); }
|
||||
.metric-card.alert .mvalue { color: var(--bad); }
|
||||
|
||||
/* ------------------------------------------------------------ 图表 */
|
||||
.chart-toolbar { display: flex; align-items: center; gap: 14px; flex-wrap: wrap; margin: 16px 0 8px; }
|
||||
.chart-tabs { display: flex; gap: 6px; flex-wrap: wrap; }
|
||||
.chart-tab {
|
||||
border: 1px solid var(--line); background: var(--transparent, transparent);
|
||||
color: var(--dim); border-radius: 999px; padding: 4px 12px; font-size: 12px;
|
||||
}
|
||||
.chart-tab.active { border-color: var(--accent); color: var(--text); background: var(--panel2); }
|
||||
label.inline { font-size: 12px; color: var(--dim); display: flex; gap: 6px; align-items: center; }
|
||||
label.inline input[type="checkbox"] { width: auto; }
|
||||
label.inline select, .log-toolbar select {
|
||||
background: var(--panel2); color: var(--text); border: 1px solid var(--line);
|
||||
border-radius: 6px; padding: 4px 8px; font-size: 12px;
|
||||
}
|
||||
.chart-stack { display: flex; flex-direction: column; gap: 14px; }
|
||||
.chart-figure { margin: 0; border: 1px solid var(--line); border-radius: 10px; background: #0b0f1a; padding: 10px 12px 4px; }
|
||||
.chart-figure figcaption { display: flex; gap: 10px; align-items: baseline; font-size: 12px; margin-bottom: 6px; }
|
||||
.chart-figure figcaption b { color: var(--text); font-size: 13px; }
|
||||
.chart-figure canvas { width: 100%; display: block; }
|
||||
|
||||
/* ------------------------------------------------------------ 日志 */
|
||||
.log-toolbar { display: flex; align-items: center; gap: 12px; flex-wrap: wrap; margin-bottom: 8px; }
|
||||
.log-toolbar input[type="search"] {
|
||||
background: var(--panel2); border: 1px solid var(--line); color: var(--text);
|
||||
border-radius: 6px; padding: 6px 10px; font-size: 12px; min-width: 240px;
|
||||
}
|
||||
.log-view {
|
||||
margin: 0; background: #0b0f1a; border: 1px solid var(--line); border-radius: 10px;
|
||||
padding: 10px 12px; height: 320px; overflow: auto; font-family: var(--mono);
|
||||
font-size: 12px; line-height: 1.55; white-space: pre-wrap; word-break: break-word;
|
||||
}
|
||||
.log-view .l { display: block; }
|
||||
.log-view .l.err { color: var(--bad); }
|
||||
.log-view .l.warn { color: var(--warn); }
|
||||
.log-view .l.json { color: var(--cyan); }
|
||||
.log-view .l.meta { color: var(--ok); }
|
||||
.log-view .l.dim { color: var(--gray); }
|
||||
.log-view mark { background: rgba(245,158,11,.28); color: inherit; border-radius: 2px; }
|
||||
|
||||
/* ------------------------------------------------------------ 表格 */
|
||||
.table-wrap { border: 1px solid var(--line); border-radius: 10px; overflow: auto; max-height: 300px; }
|
||||
table { width: 100%; border-collapse: collapse; font-size: 12px; }
|
||||
th, td { text-align: left; padding: 6px 10px; border-bottom: 1px solid var(--line); white-space: nowrap; }
|
||||
th { position: sticky; top: 0; background: var(--panel2); color: var(--dim); font-weight: 500; }
|
||||
td { font-family: var(--mono); }
|
||||
tbody tr:hover { background: rgba(79,140,255,.06); }
|
||||
|
||||
@media (max-width: 760px) {
|
||||
.topstatus { margin-left: 0; }
|
||||
.progress-row { gap: 12px; }
|
||||
}
|
||||
Reference in New Issue
Block a user