holy fuking shit

This commit is contained in:
Horldsence
2026-09-12 01:56:34 +08:00
parent 36f354736c
commit d975309a81
83 changed files with 2748 additions and 896 deletions
+5 -6
View File
@@ -126,10 +126,10 @@ class CandidateRegion:
@classmethod
def initial(cls, station: Vec2, config: ProblemConfig) -> "CandidateRegion":
"""首个 direction 后的基础集:Ω ∩ B(station, r_max) 的保守外包。"""
field, sides = config.field, config.search.polygon_sides
slack = field.omega_radius_m * (1 - math.cos(math.pi / sides))
region = cls.disk((0.0, 0.0), field.omega_radius_m, sides, slack)
region.clip_disk(station, field.r_eff_max_m, sides)
field_cfg, sides = config.field, config.search.polygon_sides
slack = field_cfg.omega_radius_m * (1 - math.cos(math.pi / sides))
region = cls.disk((0.0, 0.0), field_cfg.omega_radius_m, sides, slack)
region.clip_disk(station, field_cfg.r_eff_max_m, sides)
return region
# ------------------------------------------------------------ 收缩
@@ -198,8 +198,7 @@ class CandidateRegion:
if len(self.polygon) == 2:
p, q = self.polygon
return _point_segment_distance(point, p, q) <= tol_m + self.slack_m
for index in range(len(self.polygon)):
p = self.polygon[index]
for index, p in enumerate(self.polygon):
q = self.polygon[(index + 1) % len(self.polygon)]
if (q[0] - p[0]) * (point[1] - p[1]) - (q[1] - p[1]) * (point[0] - p[0]) \
< -tol_m - self.slack_m:
+7
View File
@@ -294,6 +294,13 @@ def _run(args) -> int:
if judge is not None:
_settle_local_judge(judge)
summary["robot_id"] = args.robot_id
return _finalize_run(args, summary, strategy, truth=truth, judge=judge,
judge_session=judge_session, real_budget_s=real_budget_s)
def _finalize_run(args, summary, strategy, *, truth, judge, judge_session,
real_budget_s) -> int:
"""补齐 summary 元信息、核对真值并输出;返回进程退出码。"""
if args.problem == 4:
summary["strategy"] = "rl_q4" if strategy == "rl" else strategy
else:
+1 -1
View File
@@ -32,7 +32,7 @@ class CoverageLedger:
"""逐频道覆盖账本:(频道, 网格点) -> 是否已测。"""
config: ProblemConfig = field(default_factory=ProblemConfig)
points: list[Vec2] = field(default_factory=lambda: coverage_points())
points: list[Vec2] = field(default_factory=coverage_points)
measured: set = field(default_factory=set) # {(channel, point_index)}
skipped: dict = field(default_factory=dict) # channel -> 原因
+4 -2
View File
@@ -23,8 +23,10 @@ def main(argv=None):
measurements = [replace(m, half_angle_deg=m.half_angle_deg+args.rounding_margin_deg)
for m in measurements]
result = solve(measurements)
result["error_policy"] = {"additional_half_angle_deg": args.rounding_margin_deg,
"effective_half_angles_deg": [m.half_angle_deg for m in measurements]}
result["error_policy"] = {
"additional_half_angle_deg": args.rounding_margin_deg,
"effective_half_angles_deg": [m.half_angle_deg for m in measurements],
}
except (ValueError, TypeError, KeyError, OSError) as exc:
parser.error(str(exc))
text = json.dumps(result, ensure_ascii=False, indent=2, allow_nan=False)
+9 -1
View File
@@ -133,6 +133,13 @@ def _enclosing_circle_reference(points: list[Point], eps: float = EPS):
return best_c, best_r
def _circle_covers(circle, point: Point, tolerance: float) -> bool:
"""Whether the circle already covers point; None means no circle yet."""
if circle is None:
return False
return distance(circle[0], point) <= circle[1] + tolerance
def smallest_enclosing_circle(points: list[Point], eps: float = EPS):
"""Welzl's randomized incremental algorithm, using an explicit call stack.
@@ -169,8 +176,9 @@ def smallest_enclosing_circle(points: list[Point], eps: float = EPS):
elif not returning:
stack.append((n, boundary, True))
stack.append((n-1, boundary, False))
elif circle is None or distance(circle[0], local[n-1]) > circle[1] + tolerance:
elif not _circle_covers(circle, local[n-1], tolerance):
stack.append((n-1, boundary + (local[n-1],), False))
assert circle is not None
center = (origin[0]+circle[0][0]*scale, origin[1]+circle[0][1]*scale)
# Recompute after transforming back: returned circle contains all input points
# to distance() precision, including any roundoff from the large translation.
+3 -2
View File
@@ -29,7 +29,7 @@ class LocalSimTransport(Transport):
def post(self, url, body, headers, timeout):
return self.request("POST", url, body, headers, timeout)
def request(self, method, url, body, headers, timeout=None):
def request(self, method, url, body, headers, _timeout=None):
"""验证原始请求边界;内存回环不模拟 TCP、UI 或连接限流。"""
target = urlsplit(url)
raw_target = target.path + (f"?{target.query}" if target.query else "")
@@ -57,5 +57,6 @@ class ScriptedFaultTransport(Transport):
self.inner.post(url, body, headers, timeout) # 服务器已受理
raise TransportError("injected: read timeout after server applied")
if isinstance(fault, tuple) and fault and fault[0] == "status":
return TransportResponse(int(fault[1]), b'{"accepted": false, "real_timestamp_ms": 0, "virtual_time_s": 0}')
body = b'{"accepted": false, "real_timestamp_ms": 0, "virtual_time_s": 0}'
return TransportResponse(int(fault[1]), body)
return self.inner.post(url, body, headers, timeout)
+1 -1
View File
@@ -17,7 +17,7 @@ import math
import numpy as np
from ..config import Vec2
from .constants import (LOW_MAX_STEPS, LOW_NUM_ACTIONS, LOW_OBS_DIM,
from .constants import (LOW_MAX_STEPS, LOW_NUM_ACTIONS,
LOW_REACH_M, LOW_STEP_M, POS_SCALE_M)
__all__ = ["build_low_observation", "LowLevelExecutor", "action_bearing",
-1
View File
@@ -144,7 +144,6 @@ class HierarchicalEnv(gym.Env):
problem=self.problem)
def _training_spec(self):
from ..harness import referee_spec
from ..hrl.curriculum import training_spec
return training_spec(self.train_config, self.stage,
max_real_duration_s=self.real_budget_s)
+1 -1
View File
@@ -19,7 +19,7 @@ from gymnasium import spaces
from ..config import ProblemConfig
from .constants import (LOW_MAX_STEPS, LOW_NUM_ACTIONS, LOW_OBS_DIM,
LOW_REACH_M, LOW_STEP_M, POS_SCALE_M)
LOW_REACH_M, LOW_STEP_M)
from .executor import (action_bearing, build_low_observation,
navigation_transition)
+1 -1
View File
@@ -48,7 +48,7 @@ def _load_maskable(model_path: Path, device: str, *,
if meta_path.exists() else {}
# 高层模型必须携带完整语义元数据;缺字段不能默认兼容,防止旧槽位/
# 旧奖励语义模型被静默加载。低层导航模型只校验基础特征版本。
strict = (act_dim == N_CANDIDATES)
strict = act_dim == N_CANDIDATES
if strict:
required = ("feature_version", "action_layout_version",
"reward_version", "algorithm", "problem",
+11 -9
View File
@@ -29,7 +29,7 @@ from typing import NamedTuple
import numpy as np
from ..belief import ChannelStatus
from ..config import ProblemConfig, Vec2
from ..config import Vec2
from ..protocol.rules import CLEAR_SUCCESS, DIRECTION, NEAR, NO_SIGNAL
from ..protocol.timing import move_time
from .constants import (CHANNEL_COUNT, CLEAR_OFFSET, COST_SCALE_S,
@@ -84,6 +84,8 @@ class HRLControllerMixin:
"""
problem: int
# 由宿主 BayesLoop.__init__ 初始化;此处声明让 pylint 可见该诊断计数。
no_shrink_measures: int = 0
# ------------------------------------------------------------ 候选构建
def build_candidates(self) -> list[Candidate]:
@@ -220,7 +222,7 @@ class HRLControllerMixin:
def hrl_observation(self, candidates: list[Candidate]) -> np.ndarray:
"""构造高层观测(shape=(HIGH_OBS_DIM,),训练/推理逐位一致)。"""
config = self.config
field = config.field
field_cfg = config.field
position = self.client.state.position
belief = self.belief
@@ -232,24 +234,24 @@ class HRLControllerMixin:
glob = [
position[0] / POS_SCALE_M,
position[1] / POS_SCALE_M,
min(self.client.state.virtual_time_s, field.max_virtual_duration_s)
/ field.max_virtual_duration_s,
min(self.client.state.virtual_time_s, field_cfg.max_virtual_duration_s)
/ field_cfg.max_virtual_duration_s,
cleared / 16.0,
accounted / CHANNEL_COUNT,
(self.client.state.channel - field.channel_min)
/ max(field.channel_max - field.channel_min, 1),
(self.client.state.channel - field_cfg.channel_min)
/ max(field_cfg.channel_max - field_cfg.channel_min, 1),
float(self.problem == 4),
detected_count / 16.0,
]
assert len(glob) == HIGH_GLOBAL_DIM
pending_station = np.zeros(CHANNEL_COUNT + 1, dtype=bool)
for index, channels in getattr(self, "_station_pending_map", {}).items():
for channels in getattr(self, "_station_pending_map", {}).values():
for channel in channels:
pending_station[channel] = True
channel_features: list[float] = []
for channel in range(field.channel_min, field.channel_max + 1):
for channel in range(field_cfg.channel_min, field_cfg.channel_max + 1):
ch = belief.channels[channel]
status6 = [0.0] * 6
status6[self._hrl_channel_status_index(channel)] = 1.0
@@ -363,7 +365,7 @@ class HRLControllerMixin:
outcome = StepOutcome(action=action, kind=cand.kind,
channel=cand.channel,
position_reached=tuple(self.client.state.position))
outcome.delta_virtual_s = (self.client.state.virtual_time_s - old_virtual)
outcome.delta_virtual_s = self.client.state.virtual_time_s - old_virtual
new_detections: list[int] = []
info_gains: dict[int, float] = {}
redundant = False
+5 -4
View File
@@ -237,10 +237,11 @@ class ChannelParticles:
base = i * self.width
for j in range(j0, min(j0 + _COARSE_FACTOR, self.width)):
index = base + j
if index in self.alive:
dist = math.dist(self.cell_center(index), point)
if dist <= limit_m and (best is None or dist < best):
best = dist
if index not in self.alive:
continue
dist = math.dist(self.cell_center(index), point)
if dist <= limit_m and (best is None or dist < best):
best = dist
return best
def guaranteed_no_signal(self, station: Vec2) -> bool:
+5 -4
View File
@@ -621,7 +621,8 @@ class DirectionalExtension(OmnidirectionalLoop):
self.ledger.mark_measured(channel, index)
self.belief.update_measure(channel, point, outcome.result, outcome.svd_deg)
if outcome.result == NEAR:
cert = clear_certificate(self.belief.candidate_set(channel), channel, self.config)
cert = clear_certificate(
self.belief.candidate_set(channel), channel, self.config)
if cert.certified:
self._attempt_clear(channel, point, cert)
if self._count_limit_reached():
@@ -629,7 +630,8 @@ class DirectionalExtension(OmnidirectionalLoop):
queue = self.belief.channels_by_status(ChannelStatus.DETECTED)
while queue and self.stop_reason is None:
channel = min(queue, key=lambda c: (
math.dist(self.client.state.position, self.belief.candidate_set(c).mec()[0]), c))
math.dist(self.client.state.position,
self.belief.candidate_set(c).mec()[0]), c))
queue.remove(channel)
self._localize_and_clear(channel)
if self._count_limit_reached():
@@ -672,8 +674,7 @@ class DirectionalExtension(OmnidirectionalLoop):
self.directional_observed.append(channel)
self.blocked_probes.setdefault(channel, []).append(point)
continue # 盘外无信号也可能是距离不足,不据此断言定向
else:
self.blocked_probes.pop(channel, None)
self.blocked_probes.pop(channel, None)
steps += 1
if self.belief.channels[channel].status is ChannelStatus.CLEARED:
return True
+12 -7
View File
@@ -109,13 +109,15 @@ class BayesLoop(OmnidirectionalLoop):
if cert is not None and cert.certified:
continue
region = self.belief.candidate_set(channel)
if max(math.dist(point, v) for v in region.polygon) > self.config.field.r_eff_min_m - 1e-3:
farthest = max(math.dist(point, v) for v in region.polygon)
if farthest > self.config.field.r_eff_min_m - 1e-3:
continue
radius = region.mec()[1]
vertices = region.polygon
stride = max(1, len(vertices) // self.config.bayes.mc_representatives)
radii = []
delta = self.config.field.bearing_error_half_deg + self.config.search.bearing_rounding_margin_deg
delta = (self.config.field.bearing_error_half_deg
+ self.config.search.bearing_rounding_margin_deg)
for target in vertices[::stride]:
theta = math.degrees(math.atan2(target[1]-point[1], target[0]-point[0]))
probe = CandidateRegion(list(vertices), region.slack_m)
@@ -146,7 +148,7 @@ class BayesLoop(OmnidirectionalLoop):
def _skip_station_channel(self, channel: int, index: int, point: Vec2) -> bool:
"""跳测规则(按 pf.version 缓存;幸存集单调收缩,判定单调)。"""
pf = self.particles[channel]
if not len(pf):
if not pf:
return True
cached = self._skip_cache.get((channel, index))
if cached is not None and cached[0] == pf.version:
@@ -219,7 +221,9 @@ class BayesLoop(OmnidirectionalLoop):
total += self.config.cost.switch_s * (channel != task.channel)
channel = task.channel
elif task.kind == "station" and task.station_channels:
pending = sorted(task.station_channels, key=lambda c: (c != channel, c))
pending = sorted(
task.station_channels,
key=lambda c, channel=channel: (c != channel, c))
total += self.config.cost.measure_s * len(pending)
total += self.config.cost.switch_s * (len(pending) - (channel in pending))
channel = pending[-1]
@@ -328,7 +332,7 @@ class BayesLoop(OmnidirectionalLoop):
return # 已证毕:清除/缺席后的 no_signal 与距离无关,不进 PF
pf = self.particles[channel]
pf.apply_measure(position, outcome.result, outcome.svd_deg)
if not len(pf):
if not pf:
if belief.status is ChannelStatus.UNKNOWN:
self.belief.mark_absent(channel)
if channel not in self.pf_absent_channels:
@@ -350,7 +354,8 @@ class BayesLoop(OmnidirectionalLoop):
limit = (self.config.field.clear_radius_m - pf.margin_m
- self.config.search.clear_margin_m)
if radius <= limit:
return self._near_clear_target(ClearCertificate(channel, center, radius, limit, True))
return self._near_clear_target(
ClearCertificate(channel, center, radius, limit, True))
return cert
def _near_clear_target(self, cert: ClearCertificate) -> ClearCertificate:
@@ -368,7 +373,7 @@ class BayesLoop(OmnidirectionalLoop):
"""定位测量点:期望 log 幸存占比 + λ·移动时间 最小的候选。"""
bayes, search = self.config.bayes, self.config.search
pf = self.particles[channel]
if not len(pf):
if not pf:
return None
region = self.belief.candidate_set(channel)
center, radius = pf.mec()
+1 -3
View File
@@ -16,8 +16,6 @@
from __future__ import annotations
from pathlib import Path
from ..belief import ChannelStatus
from ..config import ProblemConfig, Vec2
from ..coverage import CoverageLedger, coverage_points
@@ -116,7 +114,7 @@ class HRLLoop(HRLControllerMixin, BayesLoop):
if belief.status not in (ChannelStatus.CLEARED, ChannelStatus.ABSENT):
if not (self.problem == 4 and outcome.result == NO_SIGNAL):
pf.apply_measure(position, outcome.result, outcome.svd_deg)
if not len(pf):
if not pf:
if belief.status is ChannelStatus.UNKNOWN:
self.belief.mark_absent(channel)
if channel not in self.pf_absent_channels:
+175
View File
@@ -0,0 +1,175 @@
"""跨平台子进程控制:启动隔离、协作式暂停/停止、必要时强杀整棵进程树。
训练前端要在 POSIX 与 Windows 上表现一致,而两边的进程模型差别正好卡在
「停止训练」这条路径上:
- ``os.killpg`` / ``os.getpgid`` **只存在于 POSIX**;Windows 上访问它们会抛
``AttributeError``。旧实现只捕获 ``ProcessLookupError`` / ``PermissionError``,
于是 ``POST /api/train/stop`` 在 Windows 上直接 500,训练进程照跑——表现就是
「点停止没反应」。
- ``subprocess.Popen(start_new_session=True)`` 在 Windows 上被**静默忽略**:
CPython 的 Windows ``_execute_child`` 形参名就是 ``unused_start_new_session``,
要隔离进程组只能传 ``CREATE_NEW_PROCESS_GROUP``。
- POSIX 的 SIGTERM 在 Windows 上经 ``os.kill`` 会退化成 ``TerminateProcess``:
硬杀,训练脚本里"收到信号后保存模型"的处理器根本不会执行。
所以「优雅暂停 / 优雅停止」不依赖信号量语义,而是走一个**控制文件**:前端原子
写 JSON,训练脚本每步轮询。两端都只是普通文件 IO,Windows 与 POSIX 行为一致;
信号与强杀退居兜底,只在训练进程不响应控制文件时使用。
"""
from __future__ import annotations
import json
import os
import signal
import subprocess
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Any
__all__ = [
"IS_WINDOWS", "ControlState", "spawn_kwargs", "write_control",
"read_control", "write_json_atomic", "terminate_process_tree",
]
# 运行期常量而非"导入期分支":调用方(含测试)可以 monkeypatch 它来模拟 Windows
IS_WINDOWS = os.name == "nt"
# 只有 Windows 定义 CREATE_NEW_PROCESS_GROUP;POSIX 上取 0 表示不使用 creationflags
_CREATE_NEW_PROCESS_GROUP = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)
# 发出「停止」到强杀之间留给训练脚本收尾(保存模型 + 写元数据)的秒数
DEFAULT_GRACE_S = 15.0
def spawn_kwargs() -> dict[str, Any]:
"""返回 ``Popen`` 的平台相关参数:让训练进程独立于前端进程组。
POSIX 用 ``setsid``(``start_new_session``);Windows 上用
``CREATE_NEW_PROCESS_GROUP``,否则前端所在控制台的 Ctrl+C 会连带打到训练进程。
"""
if IS_WINDOWS:
if _CREATE_NEW_PROCESS_GROUP:
return {"creationflags": _CREATE_NEW_PROCESS_GROUP}
return {}
return {"start_new_session": True}
@dataclass(frozen=True)
class ControlState:
"""控制文件的一次快照;缺失或损坏时字段全为默认(即"无指令")。"""
paused: bool = False
stop: bool = False
seq: int = 0
ts: float = 0.0
def write_control(path: Path, *, paused: bool = False, stop: bool = False,
seq: int | None = None) -> ControlState:
"""原子写控制文件(同目录临时文件 + ``os.replace``)。
训练侧随时可能来读,必须避免读到写了一半的 JSON。``os.replace`` 在
Windows 与 POSIX 上都是原子的(同目录、同卷)。
"""
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
state = ControlState(
paused=bool(paused),
stop=bool(stop),
seq=int(seq) if seq is not None else time.time_ns(),
ts=time.time(),
)
payload = {"paused": state.paused, "stop": state.stop,
"seq": state.seq, "ts": state.ts}
tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp")
tmp.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
os.replace(tmp, path)
return state
def read_control(path: Path) -> ControlState:
"""读控制文件;缺失 / 半截 / 损坏一律当作"无指令",绝不让训练崩在控制面上。"""
try:
raw = Path(path).read_text(encoding="utf-8")
payload = json.loads(raw)
except (OSError, ValueError, UnicodeDecodeError):
return ControlState()
if not isinstance(payload, dict):
return ControlState()
return ControlState(
paused=bool(payload.get("paused", False)),
stop=bool(payload.get("stop", False)),
seq=int(payload.get("seq") or 0),
ts=float(payload.get("ts") or 0.0),
)
def write_json_atomic(path: Path, payload: Any) -> None:
"""原子写 JSON 文件(同目录临时文件 + ``os.replace``)。
用于"急停恢复"相关的标记文件(``resume.json`` / ``forced_stop.json`` 等):
读取方随时可能来读,必须避免看到写了一半的内容。
"""
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp")
tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2),
encoding="utf-8")
os.replace(tmp, path)
def terminate_process_tree(process: subprocess.Popen | None, *,
grace_s: float = DEFAULT_GRACE_S,
poll_s: float = 0.2) -> bool:
"""等待 ``grace_s`` 让协作式停止生效,超时则强杀整棵进程树。
返回**是否动用了强杀**:``False`` 表示进程在宽限期内自行退出(模型有机会
落盘),``True`` 表示进程被强杀、最后一段产物可能缺失。调用方应先通过控制
文件请求停止,再调用本函数;本函数不负责写控制文件。
"""
if process is None:
return False
if process.poll() is not None:
return False
deadline = time.monotonic() + max(0.0, grace_s)
while time.monotonic() < deadline:
if process.poll() is not None:
return False
time.sleep(min(poll_s, max(0.0, deadline - time.monotonic())))
if process.poll() is not None:
return False
_hard_kill_tree(process)
try:
process.wait(timeout=10.0)
except subprocess.TimeoutExpired:
pass
return True
def _hard_kill_tree(process: subprocess.Popen) -> None:
"""强杀进程树:POSIX 走进程组 SIGKILL,Windows 走 ``taskkill /T``。
这里所有 POSIX 专属调用都走 ``getattr`` 探测,Windows 上拿不到分支就退回
``Popen.kill()``——不会再出现 ``module 'os' has no attribute 'killpg'``。
"""
if IS_WINDOWS:
try:
subprocess.run(["taskkill", "/F", "/T", "/PID", str(process.pid)],
capture_output=True, check=False)
return
except OSError:
process.kill()
return
killpg = getattr(os, "killpg", None)
getpgid = getattr(os, "getpgid", None)
sigkill = getattr(signal, "SIGKILL", None)
if killpg is not None and getpgid is not None and sigkill is not None:
try:
killpg(getpgid(process.pid), sigkill)
return
except (ProcessLookupError, PermissionError, OSError):
pass
process.kill()
+3 -2
View File
@@ -13,6 +13,7 @@
from __future__ import annotations
import json
import math
import uuid
from dataclasses import dataclass
from time import monotonic, sleep
@@ -91,7 +92,7 @@ def _validate_position(position: dict, limit: float) -> None:
value = position[key]
if isinstance(value, bool) or not isinstance(value, (int, float)):
raise ValueError(f"position.{key} 必须是数值")
if value != value or value in (float("inf"), float("-inf")):
if not math.isfinite(value):
raise ValueError(f"position.{key} 必须是有限数值")
if abs(value) > limit:
raise ValueError(f"position.{key} 绝对值不得超过 {limit} 米")
@@ -234,7 +235,7 @@ class SimulatorClient:
f"剩余现实时间 {self.real_remaining_s:.1f}s 不足以预留 {margin:.1f}s 退出余量")
# ---------------------------------------------------------------- 内部机制
def _payload(self, path: str, *, position: Vec2 | None = None,
def _payload(self, _path: str, *, position: Vec2 | None = None,
channel: int | None = None) -> dict:
payload = {"arena_id": self.session.arena_id,
"robot_id": self.robot_id,
+14 -12
View File
@@ -27,7 +27,7 @@ from ..config import ProblemConfig
from ..planner import clear_certificate, fallback_clear_cells, safe_disk_radius
from ..protocol.client import SimulatorClient
from ..protocol.log import SessionLog
from ..protocol.rules import CLEAR_SUCCESS, DIRECTION, NEAR, NO_SIGNAL
from ..protocol.rules import CLEAR_SUCCESS, DIRECTION, NO_SIGNAL
from .features import NUM_ACTIONS, OBS_DIM, action_mask, build_observation, candidate_actions
@@ -65,9 +65,9 @@ class LocalizationEnv(gym.Env):
from referee import LocalSimulator as _RefereeLocalSimulator
from referee import Source
from ..harness import LocalSimTransport, referee_spec
self._Source = Source
self._LocalSimulator = _RefereeLocalSimulator
self._LocalSimTransport = LocalSimTransport
self._source_cls = Source
self._local_simulator_cls = _RefereeLocalSimulator
self._local_sim_transport_cls = LocalSimTransport
self._referee_spec = referee_spec
self._scene_rng = Random()
@@ -106,23 +106,22 @@ class LocalizationEnv(gym.Env):
# 首次检测点:距离 [300, min(1450, r_eff-20)] 内随机,定向源取朝向
# ±89.9°(接近但不跨越90°边界,保证direction)。机器狗可到Ω外 [题面]。
max_distance = max(310.0, min(1450.0, r_eff - 20.0))
if max_distance < 310.0:
max_distance = 310.0
max_distance = max(max_distance, 310.0)
probe_distance = rng.uniform(300.0, max_distance)
probe_bearing = (heading + rng.uniform(-89.9, 89.9)
if directional else rng.uniform(0.0, 360.0))
probe_radians = math.radians(probe_bearing)
probe = (x + probe_distance * math.cos(probe_radians),
y + probe_distance * math.sin(probe_radians))
source = self._Source(channel=channel, x=x, y=y, r_eff_m=r_eff,
directional=directional, heading_deg=heading)
source = self._source_cls(channel=channel, x=x, y=y, r_eff_m=r_eff,
directional=directional, heading_deg=heading)
return source, probe
def _make_episode(self) -> bool:
"""重建模拟器并完成首次测向;返回 False 表示连续未得到 direction。"""
for _ in range(20):
self._source, probe = self._make_source_and_probe()
self._simulator = self._LocalSimulator(
self._simulator = self._local_simulator_cls(
[self._source], robot_id="rl-train",
spec=self._referee_spec(
self.config, max_real_duration_s=self.real_budget_s),
@@ -131,7 +130,7 @@ class LocalizationEnv(gym.Env):
stress_mode=True)
self._client = SimulatorClient(
"rl-train", config=self.config,
transport=self._LocalSimTransport(self._simulator),
transport=self._local_sim_transport_cls(self._simulator),
log=SessionLog())
self._client.enter()
outcome = self._client.measure(probe, self._source.channel)
@@ -149,7 +148,9 @@ class LocalizationEnv(gym.Env):
if self._scene_rng.random() < self.handoff_probability:
angle = self._scene_rng.uniform(0, 2*math.pi)
reach = self.config.field.omega_radius_m * math.sqrt(self._scene_rng.random())
other = self.config.field.channel_min if self._channel != self.config.field.channel_min else self.config.field.channel_max
other = (self.config.field.channel_min
if self._channel != self.config.field.channel_min
else self.config.field.channel_max)
self._client.measure((reach*math.cos(angle), reach*math.sin(angle)), other)
self._start_virtual_s = self._client.state.virtual_time_s
return True
@@ -325,5 +326,6 @@ class LocalizationEnv(gym.Env):
return observation, float(reward), self._terminated, self._truncated, info
def close(self):
if self._client is not None and self._client.state.entered and not self._client.state.exited:
if (self._client is not None and self._client.state.entered
and not self._client.state.exited):
self._client.exit()
+3 -2
View File
@@ -19,7 +19,7 @@ import math
import numpy as np
from ..config import ProblemConfig, Vec2
from ..planner import candidate_points, safe_disk_radius
from ..planner import safe_disk_radius
from ._constants import GLOBAL_DIM, LADDER_SLOTS, NUM_ACTIONS, OBS_DIM, PER_ACTION_DIM
__all__ = [
@@ -100,7 +100,8 @@ def action_mask(region, position: Vec2, config: ProblemConfig, problem: int,
Q3有距离保证盘时屏蔽盘外探点;Q4允许盘外探点,但绝不把距离保证
当作朝向保证。训练与推理共用此函数,不读取源位置或朝向。
"""
key = lambda p: (round(p[0]*2), round(p[1]*2))
def key(p):
return (round(p[0]*2), round(p[1]*2))
visited = {key(r['position']) for r in getattr(belief, 'measurements', [])}
seen = set()
center, _ = region.mec()
+5 -2
View File
@@ -40,7 +40,8 @@ class PointSelectionPolicy:
device = "cpu"
self.device = device
meta_path = path.with_suffix('.json')
self.metadata = json.loads(meta_path.read_text(encoding='utf-8')) if meta_path.exists() else {}
self.metadata = (json.loads(meta_path.read_text(encoding='utf-8'))
if meta_path.exists() else {})
self.feature_version = self.metadata.get('feature_version', 1)
if self.feature_version not in (1, 2):
raise RuntimeError('不支持的RL特征版本')
@@ -78,7 +79,9 @@ class PointSelectionPolicy:
region, position, channel, config, problem, blocked, belief,
virtual_time_s=virtual_time_s,
receiver_channel=receiver_channel if self.feature_version >= 2 else None)
kwargs = {'action_masks': action_mask(region, position, config, problem, blocked, belief)} if self.masked else {}
kwargs = ({'action_masks': action_mask(region, position, config, problem,
blocked, belief)}
if self.masked else {})
action, _state = self.model.predict(observation, deterministic=True, **kwargs)
action = int(action)
candidates = candidate_actions(region, position, config, blocked)
+397
View File
@@ -0,0 +1,397 @@
"""训练快照的纯函数派生:曲线统计、收敛判读、进程视图。
从 ``server.py`` 拆出来的两个理由:
1. 断开 ``demo`` ↔ ``server`` 的循环导入——离线预览(``demo.py``)只想复用
这两个纯函数,没必要反向 import 整个 HTTP 服务器模块;
2. 让"数值判读"可以脱离 HTTP 与子进程单独测试。
本模块只依赖标准库,不做任何 IO、不持有任何状态。
"""
from __future__ import annotations
import math
import re
import subprocess
import time
from typing import Any
__all__ = ["MAX_LOG_LINES", "MAX_METRICS", "finite_float",
"summarize_convergence", "describe_process", "process_pid"]
MAX_LOG_LINES = 4000
MAX_METRICS = 5000
_NUM_RE = re.compile(r"^[+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?$")
def finite_float(value: Any) -> float | None:
"""把任意输入转成有限浮点数;NaN / ±inf / 非数字一律返回 None。
旧实现用 ``value != value`` 判 NaN,pylint 报 ``comparison-with-itself``;
这里用 ``math.isfinite`` 表达同一语义,意图更直白。
"""
if value is None or isinstance(value, bool):
return None
if isinstance(value, (int, float)):
value = float(value)
if not math.isfinite(value):
return None
return value
text = str(value).strip()
if not _NUM_RE.match(text):
return None
try:
return float(text)
except ValueError:
return None
def _series(metrics: list[dict[str, Any]], key: str) -> list[tuple[float, float]]:
points: list[tuple[float, float]] = []
for point in metrics:
value = finite_float(point.get(key))
steps = finite_float(point.get("timesteps")) or finite_float(
point.get("total_timesteps"))
if value is None:
continue
points.append((steps if steps is not None else float(len(points) + 1), value))
return points
def _last_value(points: list[tuple[float, float]]) -> float | None:
return points[-1][1] if points else None
def _spread(points: list[tuple[float, float]], window: int = 10
) -> tuple[float, float] | None:
"""最近 window 个点的 (极差, 起点绝对值),用于判断"是否还在动"。"""
sample = points[-window:]
if len(sample) < 3:
return None
ys = [item[1] for item in sample]
return max(ys) - min(ys), abs(ys[0])
def _cv(points: list[tuple[float, float]], window: int = 10) -> float | None:
"""最近 window 个点的变异系数(标准差/|均值|),越小越平稳。"""
sample = [item[1] for item in points[-window:]]
if len(sample) < 3:
return None
mean = sum(sample) / len(sample)
if abs(mean) < 1e-12:
return None
var = sum((value - mean) ** 2 for value in sample) / len(sample)
return (var ** 0.5) / abs(mean)
def _slope(points: list[tuple[float, float]], window: int = 8) -> float | None:
sample = points[-window:]
if len(sample) < 3:
return None
xs = [item[0] for item in sample]
ys = [item[1] for item in sample]
n = len(xs)
mean_x = sum(xs) / n
mean_y = sum(ys) / n
denom = sum((x - mean_x) ** 2 for x in xs)
if abs(denom) < 1e-12:
return None
return sum((x - mean_x) * (y - mean_y) for x, y in zip(xs, ys)) / denom
def _rate_per_1k(reward_points: list[tuple[float, float]],
window: int = 8) -> float | None:
"""最近 window 个点的奖励增益换算成"每 1000 环境步",便于读趋势。"""
slope = _slope(reward_points, window)
if slope is None:
return None
return slope * 1000.0
def _fmt_num(value: float | None, digits: int = 4) -> str:
if value is None:
return "—"
if value != 0 and abs(value) < 1e-3:
return f"{value:.3e}"
return f"{value:.{digits}g}"
def _flatness(points: list[tuple[float, float]], window: int = 10
) -> tuple[float, float] | None:
"""窗口内"趋势位移 / 逐点噪声"比值:<1 表示看不出系统性变化。
判定收敛时只看斜率绝对值会把"很慢但持续的提升"误判为平台;这里把窗口
对半切,用两半均值之差当趋势位移,用相邻点平均绝对差当噪声尺度。返回
(比值, 噪声尺度),两者都小才是真的"又平又稳"。
"""
sample = points[-window:]
if len(sample) < 4:
return None
ys = [item[1] for item in sample]
half = len(ys) // 2
shift = abs(sum(ys[half:]) / len(ys[half:]) - sum(ys[:half]) / len(ys[:half]))
noise = sum(abs(b - a) for a, b in zip(ys, ys[1:])) / (len(ys) - 1)
if noise <= 0:
return (0.0, 0.0) if shift <= 0 else (float("inf"), 0.0)
return shift / noise, noise
def summarize_convergence(metrics: list[dict[str, Any]],
monitor: list[dict[str, Any]]) -> dict[str, Any]:
"""给前端返回可解释的收敛状态与各参数最近值。
这是**启发式判读**,不替代正式学习曲线分析:只依据最近窗口的斜率、
极差与变异系数给出"平坦/上升/未上升"的分类,并把同一组数字交给前端
展示,便于人工复核。
"""
reward_points = _series(metrics, "ep_rew_mean")
if not reward_points and monitor:
reward_points = [(float(index), float(row.get("r", 0.0)))
for index, row in enumerate(monitor)
if finite_float(row.get("r")) is not None]
kl_points = _series(metrics, "approx_kl")
ev_points = _series(metrics, "explained_variance")
vf_points = _series(metrics, "value_loss")
pg_points = _series(metrics, "policy_gradient_loss")
entropy_points = _series(metrics, "entropy_loss")
fps_points = _series(metrics, "fps")
reward_slope = _slope(reward_points)
reward_rate = _rate_per_1k(reward_points)
spread = _spread(reward_points)
cv = _cv(reward_points)
latest_kl = _last_value(kl_points)
latest_ev = _last_value(ev_points)
latest_vf = _last_value(vf_points)
latest_pg = _last_value(pg_points)
latest_entropy = _last_value(entropy_points)
latest_fps = _last_value(fps_points)
latest_timesteps = metrics[-1].get("timesteps") if metrics else None
if latest_timesteps is None and metrics:
latest_timesteps = metrics[-1].get("total_timesteps")
latest_reward = _last_value(reward_points)
latest_len = _last_value(_series(metrics, "ep_len_mean"))
latest_updates = _last_value(_series(metrics, "n_updates"))
window = min(10, max(3, len(reward_points)))
reward_window = [round(value, 6) for _, value in reward_points[-window:]]
best_reward = max((value for _, value in reward_points), default=None)
notes: list[str] = []
if not metrics and not monitor:
status, tone = "等待首个 rollout", "gray"
notes.append("训练进程启动后先输出环境初始化与首个 rollout 的进度行。")
elif not metrics:
status, tone = "已收到回合奖励,等待进度行", "warn"
notes.append("Monitor CSV 已有回合记录,但尚未收到 --progress-json 进度行;"
"首个 rollout 结束后会出现完整指标。")
elif len(metrics) < 3:
status, tone = "预热阶段", "warn"
notes.append("已收到初始指标,需要更多 rollout 才能判断趋势。")
else:
tone, status = "ok", "收敛趋势良好"
# 平台判据(三条同时成立):对半窗口看不出位移、窗口内波动小、
# 逐点噪声本身也小。慢速但持续的提升会因第一条被排除。
flat = _flatness(reward_points)
plateau = False
if flat is not None and cv is not None and latest_reward is not None:
flat_ratio, flat_noise = flat
scale = max(abs(latest_reward), 1e-9)
plateau = (flat_ratio <= 1.0 and cv <= 0.15
and flat_noise <= 0.02 * scale
and (spread is None or spread[1] < 1e-12
or spread[0] <= 0.15 * spread[1]))
if latest_kl is not None and latest_kl > 0.05:
status, tone = "KL 偏大,策略更新可能过猛", "bad"
notes.append(f"approx_kl={latest_kl:.4f} > 0.05")
elif latest_ev is not None and latest_ev < 0.0:
status, tone = "价值函数尚未拟合", "warn"
notes.append(f"explained_variance={latest_ev:.3f} < 0")
elif reward_slope is not None and reward_slope < 0:
status, tone = "奖励未形成上升趋势", "warn"
notes.append(f"最近 {window} 点奖励斜率为负({_fmt_num(reward_slope)}/step)")
elif plateau:
status, tone = "已趋于平稳(疑似收敛)", "ok"
notes.append("窗口内对半位移小于逐点噪声,且波动很小"
f"(变异系数 {_fmt_num(cv, 3)})")
else:
notes.append(f"最近 {window} 点奖励斜率为正({_fmt_num(reward_slope)}/step)")
if latest_kl is not None and latest_kl <= 0.02:
notes.append(f"approx_kl={latest_kl:.4f} 较稳定")
if latest_ev is not None:
notes.append(f"explained_variance={latest_ev:.3f}")
if cv is not None:
notes.append(f"奖励变异系数={_fmt_num(cv, 3)}")
if latest_entropy is not None and abs(latest_entropy) < 1e-3:
notes.append("策略熵接近 0,探索可能过早停止")
return {
"status": status,
"tone": tone,
"notes": notes,
"window": window,
"latest_timesteps": latest_timesteps,
"reward_slope": reward_slope,
"reward_rate_per_1k": reward_rate,
"reward_cv": cv,
"reward_spread": spread[0] if spread else None,
"reward_window": reward_window,
"best_reward": best_reward,
"latest_reward": latest_reward,
"latest_ep_len": latest_len,
"latest_updates": latest_updates,
"latest_kl": latest_kl,
"latest_explained_variance": latest_ev,
"latest_value_loss": latest_vf,
"latest_policy_loss": latest_pg,
"latest_entropy_loss": latest_entropy,
"latest_fps": latest_fps,
"reward_points": len(reward_points),
}
# 快照状态 -> 界面展示用的进程语义(阶段名称、色调、是否终态)
_STATUS_VIEW = {
"created": ("已创建", "gray", False, "命令已构造,等待 spawn"),
"running": ("运行中", "ok", False, "子进程持有控制台输出与独立进程组"),
"paused": ("已暂停", "warn", False, "进程存活但停在 step 边界;点「继续」从暂停处恢复"),
"stopping": ("停止中", "warn", False, "已请求优雅停止,等待子进程保存并退出"),
"stopped": ("已手动停止", "warn", True, "收到停止请求后退出;脚本会在 step 边界保存模型与元数据"),
"exited": ("正常结束", "ok", True, "退出码 0:训练与评估流程走完"),
"failed": ("异常结束", "bad", True, "非零退出码或启动失败,见日志尾部"),
}
_PHASES = [
("created", "构造命令"),
("spawn", "启动子进程"),
("rollout", "采集 rollout"),
("update", "PPO 更新"),
("evaluate", "评估 / 保存"),
("done", "收敛归档"),
]
# 状态 -> 当前所处的阶段节点;节点之前为已完成,之后为未开始。
_STATUS_PHASE = {
"created": "created",
"running": "update",
"paused": "update",
"stopping": "update",
"stopped": "done",
"exited": "done",
"failed": "done",
}
def process_pid(process: subprocess.Popen | None) -> int | None:
"""已退出进程的 pid 访问会抛 OSError;这里统一吞掉,只用于展示。"""
if process is None:
return None
try:
return int(process.pid)
except (OSError, ValueError):
return None
def describe_process(snapshot: dict[str, Any]) -> dict[str, Any]:
"""把 snapshot 派生为界面直接可渲染的进程对象。
包含:状态语义词、进度(步/百分比)、墙钟与剩余时间估计、最近吞吐、
进程时间线节点。数字都取自 snapshot,前端不再重复累加,避免两个刷新
频率不同的计数器互相打架。
"""
status = str(snapshot.get("status") or "created")
label, tone, terminal, hint = _STATUS_VIEW.get(
status, (status, "gray", False, ""))
config = snapshot.get("config") or {}
order = [key for key, _ in _PHASES]
current = _STATUS_PHASE.get(status, "created")
timeline = []
for key, title in _PHASES:
index = order.index(key)
if index < order.index(current):
state = "done"
elif index == order.index(current):
state = "active"
else:
state = "pending"
timeline.append({"key": key, "label": title, "state": state})
metrics = snapshot.get("metrics") or []
latest: dict[str, Any] = metrics[-1] if metrics else {}
convergence = snapshot.get("convergence") or {}
total_steps = finite_float(config.get("timesteps")) or 0.0
# SB3 表格行可能没有 steps 字段,因此进度取"所有进度点的最大步数",
# 而不是只看最后一个点——否则末行恰好缺字段时进度会回退到 0。
done_steps = 0.0
for point in metrics:
value = (finite_float(point.get("timesteps"))
or finite_float(point.get("total_timesteps")))
if value is not None and value > done_steps:
done_steps = value
if done_steps == 0.0:
done_steps = finite_float(convergence.get("latest_timesteps")) or 0.0
# init/resume(课程迁移)时 num_timesteps 从上一阶段继续,进度必须按本次
# 增量算;start_timesteps 由训练脚本的进度 JSON 提供,缺省按 0 处理。
start_steps = 0.0
for point in metrics:
value = finite_float(point.get("start_timesteps"))
if value is not None:
start_steps = value
break
trained_steps = max(0.0, done_steps - start_steps)
percent = (100.0 * trained_steps / total_steps) if total_steps > 0 else None
if percent is not None:
percent = max(0.0, min(100.0, percent))
elapsed = float(snapshot.get("elapsed_s") or 0.0)
rate = _latest_rate(metrics)
remaining: float | None = None
eta_ts: float | None = None
if rate and rate > 0 and total_steps > 0 and trained_steps < total_steps:
remaining = (total_steps - trained_steps) / rate
eta_ts = float(snapshot.get("finished_at") or time.time()) + remaining
if not terminal and remaining is None:
cap = finite_float(config.get("max_wall_s")) or 0.0
if cap > 0:
remaining = max(0.0, cap - elapsed)
return {
"label": label,
"tone": tone,
"terminal": terminal,
"hint": hint,
"phase": current,
"timeline": timeline,
"percent": percent,
"done_steps": done_steps,
"start_steps": start_steps,
"trained_steps": trained_steps,
"total_steps": total_steps,
"elapsed_s": round(elapsed, 3),
"rate_per_s": rate,
"remaining_s": remaining,
"eta_epoch_s": eta_ts,
"log_total": snapshot.get("log_total"),
"log_capacity": MAX_LOG_LINES,
"metric_total": snapshot.get("metric_total"),
"latest": latest,
}
def _latest_rate(metrics: list[dict[str, Any]], window: int = 6) -> float | None:
"""最近若干个**带 fps 的**进度点的平均吞吐(步/秒),用于剩余时间估计。
progress JSON 行不含 fps,只有 SB3 表格行有;若只看最后 window 个点,
恰好落在若干 JSON 行之间时会得到 None,剩余时间就会退化成墙钟上限的
粗糙估计。这里向前扫描直到拿到至少 1 个 fps(最多看 3*window 个点)。
"""
if not metrics:
return None
usable: list[float] = []
for point in reversed(metrics[-window * 3:]):
value = finite_float(point.get("fps"))
if value and value > 0:
usable.append(value)
if len(usable) >= window:
break
if not usable:
return None
return sum(usable) / len(usable)
+15 -3
View File
@@ -15,6 +15,8 @@ import random
import time
from typing import Any
from .analysis import describe_process, summarize_convergence
__all__ = ["DEMO_CONFIG", "DEMO_SCENARIOS", "demo_metrics", "demo_monitor",
"demo_log_lines", "demo_snapshot"]
@@ -44,7 +46,7 @@ DEMO_STEPS = 48
def _finite(value: float) -> float | None:
if value != value or value in (float("inf"), float("-inf")):
if not math.isfinite(value):
return None
return round(value, 8)
@@ -171,8 +173,6 @@ def demo_snapshot(*, log_tail: int = 400, scenario: str = "rising",
(而不是刚好 100%);``scenario`` 选择收敛判读分支;``fps`` 决定剩余
时间估计的吞吐量级。
"""
from .server import describe_process, summarize_convergence
now = time.time()
ratio = min(1.0, max(0.05, float(done_ratio)))
metrics = demo_metrics(count=max(6, int(round(DEMO_STEPS * ratio))),
@@ -193,6 +193,7 @@ def demo_snapshot(*, log_tail: int = 400, scenario: str = "rising",
"elapsed_s": 184.0,
"returncode": None,
"error": None,
"forced_kill": False,
"command": list(DEMO_COMMAND),
"model": str(config["model"]),
"config": config,
@@ -203,6 +204,17 @@ def demo_snapshot(*, log_tail: int = 400, scenario: str = "rising",
"metric_end": len(metrics),
"metric_total": len(metrics),
"monitor": monitor,
"recovery": {
"can_resume": True,
"model": str(config["model"]),
"timesteps": int(DEMO_STEPS * ratio * 12),
"reason": "recovery",
"updated_unix_s": now - 24.0,
"source": "resume.json",
"hint_file": str(config["model"]).rsplit("/", 1)[0] + "/resume.json",
"candidates": 3,
"preview": True,
},
"preview": True,
"preview_scenario": scenario,
}
+19
View File
@@ -163,6 +163,24 @@ CONFIG_FIELDS: list[dict[str, Any]] = [
"label": "覆盖已有模型 --force", "default": False,
"help": "输出路径已存在时必须勾选,否则训练脚本会拒绝覆盖。与续训互斥。",
},
# ---------------------------------------------------------- 恢复与落盘
{
"key": "recovery_every", "kind": "integer", "group": "recovery",
"label": "滚动恢复点间隔 --recovery-every", "default": 2000,
"help": "每 N 个环境步原子覆盖写 <checkpoints>/<stem>_latest.zip,"
"训练开始前先写起步基线;急停(强杀)后最多丢 N 步。0=关闭。",
},
{
"key": "checkpoint_every", "kind": "integer", "group": "recovery",
"label": "历史 checkpoint 间隔 --checkpoint-every", "default": 20000,
"help": "每 N 个环境步另留一份 <stem>_<步数>.zip 供返祖对比;"
"0=只保留滚动恢复点。",
},
{
"key": "checkpoint_keep", "kind": "integer", "group": "recovery",
"label": "历史 checkpoint 保留数 --checkpoint-keep", "default": 3,
"help": "只保留最近 K 个历史 checkpoint;滚动恢复点与起步基线不受影响。",
},
]
CONFIG_GROUPS: list[dict[str, str]] = [
@@ -170,6 +188,7 @@ CONFIG_GROUPS: list[dict[str, str]] = [
{"key": "ppo", "label": "PPO 超参数", "hint": "采样宽度与更新强度"},
{"key": "runtime", "label": "运行环境", "hint": "并行、设备与低层执行器"},
{"key": "scale", "label": "训练规模", "hint": "步数、墙钟上限与评估"},
{"key": "recovery", "label": "恢复与落盘", "hint": "急停恢复点与历史 checkpoint"},
{"key": "paths", "label": "模型与迁移", "hint": "产物路径、续训与课程迁移"},
]
+285 -401
View File
@@ -4,7 +4,8 @@
设计边界:
- 只监听 127.0.0.1,只允许白名单静态文件;
- 只允许一个训练任务,命令由服务端用参数列表构造,不经过 shell;
- 支持启动/停止/状态查询;停止时发送 SIGTERM 到独立进程组;
- 支持启动/暂停/继续/停止/状态查询;暂停与停止通过**控制文件**下发,
POSIX 与 Windows 行为一致(详见 :mod:`mcm26b.process_control`);
- 训练进程通过 ``--progress-json`` 输出单行 JSON 进度,服务端同时兼容
Stable-Baselines3 的表格输出和 Monitor CSV。
@@ -12,15 +13,17 @@
- ``GET /api/train/schema`` 字段/分组/预设/指标字典(``schema.py`` 单一出处)
- ``GET /api/train/state`` 快照;``after_metric=N`` 只回增量指标
- ``POST /api/train/start`` 以部分字段启动(其余取默认值)
- ``POST /api/train/stop`` SIGTERM 独立进程组
- ``POST /api/train/pause`` 请求训练进程在 step 边界挂起
- ``POST /api/train/resume`` 解除挂起
- ``POST /api/train/stop`` 请求优雅停止,超时后强杀进程树
"""
from __future__ import annotations
import dataclasses
import json
import os
import re
import signal
import subprocess
import sys
import threading
@@ -32,15 +35,24 @@ from pathlib import Path
from typing import Any
from urllib.parse import parse_qs, urlsplit
from ..process_control import (DEFAULT_GRACE_S, spawn_kwargs,
terminate_process_tree, write_control,
write_json_atomic)
from .analysis import (MAX_LOG_LINES, MAX_METRICS, describe_process, finite_float,
process_pid, summarize_convergence)
from .demo import DEMO_SCENARIOS, demo_snapshot
from .schema import config_schema
__all__ = ["TrainingJob", "TrainingUI", "run_training_ui", "build_command",
"summarize_convergence", "describe_process"]
"summarize_convergence", "describe_process", "discover_recovery"]
STATIC_DIR = Path(__file__).resolve().parent / "static"
REPO_ROOT = Path(__file__).resolve().parents[3]
TRAIN_SCRIPT = REPO_ROOT / "scripts" / "train_hrl_high.py"
# 控制文件集中放在 output/ 下(已 gitignore),job 之间互不干扰
CONTROL_DIR = REPO_ROOT / "output" / "hrl" / "ui" / "control"
# 从发出停止请求到强杀之间留给训练脚本收尾的秒数
STOP_GRACE_S = DEFAULT_GRACE_S
STATIC_FILES = {
"/": ("index.html", "text/html; charset=utf-8"),
"/index.html": ("index.html", "text/html; charset=utf-8"),
@@ -49,10 +61,6 @@ STATIC_FILES = {
"/style.css": ("style.css", "text/css; charset=utf-8"),
}
MAX_LOG_LINES = 4000
MAX_METRICS = 5000
_NUM_RE = re.compile(r"^[+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?$")
_SB3_SECTIONS = ("rollout/", "time/", "train/", "system/")
# 路径/字符串参数只拒绝控制字符(含 \r\n\t 与 DEL),不做 shell 元字符过滤:
# 命令走参数列表、不经过 shell,这里只是防日志与命令行被注入控制序列。
@@ -61,23 +69,6 @@ _SB3_SECTIONS = ("rollout/", "time/", "train/", "system/")
_SAFE_ARG_RE = re.compile(r"^[^\x00-\x1f\x7f]{1,512}$")
def _finite_float(value: Any) -> float | None:
if value is None or isinstance(value, bool):
return None
if isinstance(value, (int, float)):
value = float(value)
if value != value or value in (float("inf"), float("-inf")):
return None
return value
text = str(value).strip()
if not _NUM_RE.match(text):
return None
try:
return float(text)
except ValueError:
return None
def _normalize_metric(key: str) -> str:
"""把 SB3 的 ``rollout/ep_rew_mean`` 与进度 JSON 的 ``ep_rew_mean``
归一化到同一短名,便于前端绘制。"""
@@ -109,6 +100,10 @@ class TrainConfig:
max_wall_s: float = 1200.0
local_selector: str = "geometry"
local_model: str | None = None
# 落盘/恢复:默认值与 scripts/train_hrl_high.py 保持一致
checkpoint_every: int = 20000
checkpoint_keep: int = 3
recovery_every: int = 2000
verbose: int = 1
@classmethod
@@ -117,9 +112,11 @@ class TrainConfig:
raise ValueError("请求体必须是 JSON 对象")
defaults = cls()
data: dict[str, Any] = {}
for field in cls.__dataclass_fields__:
if field in payload and payload[field] is not None:
data[field] = payload[field]
# 用 dataclasses.fields 而不是 cls.__dataclass_fields__:后者是内部
# 实现细节,pylint 看不到它,也无法保证未来版本仍存在。
for field in dataclasses.fields(cls):
if field.name in payload and payload[field.name] is not None:
data[field.name] = payload[field.name]
cfg = cls(**{**defaults.__dict__, **data})
cfg.validate()
return cfg
@@ -153,6 +150,12 @@ class TrainConfig:
raise ValueError("max_wall_s 必须在 0..7200 秒")
if int(self.eval_episodes) < 0:
raise ValueError("eval_episodes 不能为负")
if not 0 <= int(self.checkpoint_every) <= 10_000_000:
raise ValueError("checkpoint_every 必须在 0..10,000,000")
if not 1 <= int(self.checkpoint_keep) <= 100:
raise ValueError("checkpoint_keep 必须在 1..100")
if not 0 <= int(self.recovery_every) <= 10_000_000:
raise ValueError("recovery_every 必须在 0..10,000,000")
for label, value in (("model", self.model), ("init", self.init),
("local_model", self.local_model)):
if value is None:
@@ -207,6 +210,9 @@ def build_command(config: TrainConfig) -> list[str]:
"--n-epochs", str(config.n_epochs),
"--max-wall-s", str(config.max_wall_s),
"--local-selector", config.local_selector,
"--checkpoint-every", str(config.checkpoint_every),
"--checkpoint-keep", str(config.checkpoint_keep),
"--recovery-every", str(config.recovery_every),
"--progress-json",
"--verbose", str(config.verbose),
]
@@ -223,25 +229,118 @@ def build_command(config: TrainConfig) -> list[str]:
return cmd
class TrainingJob:
"""单个训练子进程的生命周期、日志和指标缓存。"""
def discover_recovery(run_dir: Path, model_path: Path) -> dict[str, Any]:
"""找出"现在就能继续训练"的那份产物。
def __init__(self, config: TrainConfig):
优先级:
1. ``<run_dir>/resume.json``——训练脚本写的权威指针(校验目标文件真实存在);
2. 否则按 mtime 取最新:最终模型 / 滚动恢复点 / 历史 checkpoint / 异常兜底。
只认**已完整落盘**的文件:模型的 zip 与 json 都是原子替换写的,所以不会
出现"文件在、但打不开"的半截快照。
"""
run_dir = Path(run_dir)
model_path = Path(model_path)
hint_file = run_dir / "resume.json"
hint = _read_json_file(hint_file)
candidates: list[tuple[float, str, str]] = []
def _consider(path: Path, reason: str) -> None:
try:
stat = path.stat()
except OSError:
return
if stat.st_size > 0:
candidates.append((stat.st_mtime, str(path), reason))
_consider(model_path, "final_model")
ckpt_dir = run_dir / "checkpoints"
if ckpt_dir.is_dir():
_consider(ckpt_dir / f"{model_path.stem}_ckpt_latest.zip", "recovery")
for path in ckpt_dir.glob(f"{model_path.stem}_ckpt_[0-9]*.zip"):
if path.stem.rsplit("_", 1)[-1].isdigit():
_consider(path, "checkpoint")
_consider(run_dir / f"{model_path.stem}_emergency.zip", "emergency")
state: dict[str, Any] = {
"can_resume": False, "model": None, "timesteps": None,
"reason": "none", "updated_unix_s": None, "source": "none",
"hint_file": str(hint_file), "candidates": len(candidates),
}
if hint and hint.get("model") and Path(str(hint["model"])).is_file():
state.update({
"model": str(hint["model"]),
"timesteps": _int_or_none(hint.get("timesteps")),
"reason": str(hint.get("reason") or "resume_hint"),
"updated_unix_s": _float_or_none(hint.get("created_unix_s")),
"source": "resume.json",
})
elif candidates:
mtime, path, reason = max(candidates, key=lambda item: item[0])
state.update({
"model": path, "reason": reason,
"updated_unix_s": mtime, "source": "scan",
})
if state["model"]:
state["can_resume"] = Path(str(state["model"])).is_file()
return state
def _read_json_file(path: Path) -> dict[str, Any] | None:
"""读 JSON 文件;缺失或损坏返回 None(恢复指针不是关键路径,不能因此崩)。"""
try:
payload = json.loads(Path(path).read_text(encoding="utf-8"))
except (OSError, ValueError, UnicodeDecodeError):
return None
return payload if isinstance(payload, dict) else None
def _int_or_none(value: Any) -> int | None:
try:
return int(value)
except (TypeError, ValueError):
return None
def _float_or_none(value: Any) -> float | None:
try:
return float(value)
except (TypeError, ValueError):
return None
class TrainingJob:
"""单个训练子进程的生命周期、日志和指标缓存。
``control_dir`` 是控制文件的落地目录;默认与模型同目录,服务端则统一放到
``output/hrl/ui/control``(已被 gitignore)。
"""
def __init__(self, config: TrainConfig, control_dir: Path | None = None):
self.config = config
self.job_id = f"job-{int(time.time() * 1000)}"
self.command = config.command()
self.control_path = (Path(control_dir) if control_dir is not None
else config.model_path.parent) / \
f"{self.job_id}.control.json"
# 控制文件是暂停/停止的唯一可靠通道:Windows 上没有 SIGSTOP,
# 且 os.kill(SIGTERM) 会退化成硬杀,信号无法承载"优雅收尾"语义。
self.command = config.command() + ["--control-file", str(self.control_path)]
self.created_at = time.time()
self.started_at: float | None = None
self.finished_at: float | None = None
self.returncode: int | None = None
self.status = "created" # created/running/exited/failed/stopped
# created/running/paused/stopping/stopped/exited/failed
self.status = "created"
self.error: str | None = None
self.forced_kill = False # 停止请求超时后是否动用了强杀
self.process: subprocess.Popen | None = None
self.logs: deque[str] = deque(maxlen=MAX_LOG_LINES)
self.metrics: deque[dict[str, Any]] = deque(maxlen=MAX_METRICS)
self.monitor: list[dict[str, Any]] = []
self._lock = threading.RLock()
self._reader: threading.Thread | None = None
self._stopper: threading.Thread | None = None
self._table_sections: list[tuple[str, dict[str, Any]]] = []
self._table_steps: float | None = None # 当前表的 total_timesteps
self._pending_table: dict[str, Any] | None = None # 未见刻度的半张表
@@ -254,10 +353,13 @@ class TrainingJob:
with self._lock:
if self.process is not None and self.process.poll() is None:
raise RuntimeError("已有训练任务在运行")
# 先落一份"不暂停、不停止"的初始控制文件,训练脚本首步就能读到;
# 否则脚本要等到第一次轮询才区分"文件缺失"与"无指令"。
write_control(self.control_path, paused=False, stop=False)
env = os.environ.copy()
env["PYTHONUNBUFFERED"] = "1"
env["HRL_UI_JOB_ID"] = self.job_id
self.process = subprocess.Popen(
self.process = subprocess.Popen( # pylint: disable=consider-using-with
self.command,
cwd=str(REPO_ROOT),
stdout=subprocess.PIPE,
@@ -265,7 +367,7 @@ class TrainingJob:
text=True,
bufsize=1,
env=env,
start_new_session=True,
**spawn_kwargs(),
)
self.started_at = time.time()
self.status = "running"
@@ -273,18 +375,101 @@ class TrainingJob:
target=self._read_loop, name=f"{self.job_id}-reader", daemon=True)
self._reader.start()
def stop(self) -> bool:
def pause(self) -> bool:
"""请求训练进程在下一个 step 边界挂起(跨平台:控制文件)。"""
with self._lock:
process = self.process
if process is None or process.poll() is not None:
return False
try:
os.killpg(os.getpgid(process.pid), signal.SIGTERM)
except (ProcessLookupError, PermissionError):
process.terminate()
self.status = "stopping"
if self.status not in ("running", "paused"):
return False
write_control(self.control_path, paused=True, stop=False)
self.status = "paused"
return True
def resume(self) -> bool:
"""解除挂起;训练从暂停的那个 step 继续。"""
with self._lock:
process = self.process
if process is None or process.poll() is not None:
return False
if self.status not in ("paused", "running"):
return False
write_control(self.control_path, paused=False, stop=False)
self.status = "running"
return True
def stop(self) -> bool:
"""请求优雅停止;超时后由看门狗强杀整棵进程树。
控制文件先于信号:Windows 上 SIGTERM 等价于 TerminateProcess,训练
脚本"保存模型 + 写元数据"的收尾分支不会执行;控制文件则两个平台都能
命中同一段协作式收尾。强杀放到独立线程,避免 HTTP 请求被 grace 等待卡住。
"""
with self._lock:
process = self.process
if process is None or process.poll() is not None:
return False
write_control(self.control_path, paused=False, stop=True)
self.status = "stopping"
if self._stopper is None or not self._stopper.is_alive():
self._stopper = threading.Thread(
target=self._stop_watchdog, args=(process,),
name=f"{self.job_id}-stopper", daemon=True)
self._stopper.start()
return True
def _stop_watchdog(self, process: subprocess.Popen) -> None:
"""等 grace 让协作式停止生效,超时强杀并记下恢复点。"""
forced = terminate_process_tree(process, grace_s=STOP_GRACE_S)
with self._lock:
self.forced_kill = forced
if forced and not self.error:
self.error = (f"训练进程未在 {STOP_GRACE_S:.0f}s 内响应停止请求,"
f"已强制结束进程树")
if forced:
self._write_forced_stop(process.pid)
def _write_forced_stop(self, pid: int) -> None:
"""强杀后落一份 `<模型目录>/forced_stop.json`,写明"从哪继续"。
强杀没有给训练脚本任何收尾机会,所以这里由前端补上审计与恢复指引:
磁盘上仍有原子落盘的滚动恢复点 / 历史 checkpoint,``recovery.model``
就是可直接 ``--resume`` 的目标。
"""
run_dir = self.config.model_path.parent
try:
recovery = discover_recovery(run_dir, self.config.model_path)
write_json_atomic(run_dir / "forced_stop.json", {
"event": "forced_stop",
"job_id": self.job_id,
"pid": int(pid),
"created_unix_s": time.time(),
"reason": "grace_timeout",
"grace_s": STOP_GRACE_S,
"recovery": recovery,
"note": ("训练进程未响应控制文件,已强杀整棵进程树;"
"把 recovery.model 作为 --resume 目标即可继续"
"(timesteps 为追加步数)"),
})
except OSError as exc: # 标记文件写不了也不能影响停止流程
with self._lock:
if not self.error:
self.error = f"强杀后写恢复标记失败:{exc}"
def wait_for_exit(self, timeout: float | None = None) -> bool:
"""等待停止流程收尾(优雅退出或强杀);供前端进程退出前调用。
训练子进程在独立进程组里,前端先退出并不会带走它。若不在这里等,
前端一退出,子进程会成为孤儿——虽然控制文件仍会让它自行收尾,但
日志与退出码就没人接了。
"""
thread = self._stopper
if thread is not None:
thread.join(timeout)
process = self.process
return process is None or process.poll() is not None
def _read_loop(self) -> None:
process = self.process
assert process is not None and process.stdout is not None
@@ -315,10 +500,38 @@ class TrainingJob:
def _on_line(self, line: str) -> None:
with self._lock:
self.logs.append(line)
self._sync_control_events(line)
self._parse_progress_json(line)
self._parse_sb3_table(line)
self._refresh_monitor()
def _sync_control_events(self, line: str) -> None:
"""用训练脚本的回显校正前端状态。
正常情况下状态由前端自己设置;但外部直接改控制文件、或脚本比前端更早
感知到指令时,以脚本回显的 ``paused`` / ``resumed`` / ``stop_requested``
为准更真实。
"""
stripped = line.strip()
if not stripped.startswith("{") or '"event"' not in stripped:
return
try:
payload = json.loads(stripped)
except ValueError:
return
if not isinstance(payload, dict):
return
event = payload.get("event")
if event not in ("paused", "resumed", "stop_requested"):
return
with self._lock:
if event == "paused" and self.status == "running":
self.status = "paused"
elif event == "resumed" and self.status == "paused":
self.status = "running"
elif event == "stop_requested" and self.status in ("running", "paused"):
self.status = "stopping"
# ------------------------------------------------------------ parsers
def _parse_progress_json(self, line: str) -> None:
stripped = line.strip()
@@ -332,9 +545,9 @@ class TrainingJob:
return
point = {_normalize_metric(str(key)): value
for key, value in payload.items() if key != "event"}
steps = _finite_float(point.get("timesteps"))
steps = finite_float(point.get("timesteps"))
if steps is None:
steps = _finite_float(point.get("total_timesteps"))
steps = finite_float(point.get("total_timesteps"))
if steps is not None:
with self._lock:
self._last_json_steps = steps
@@ -371,7 +584,7 @@ class TrainingJob:
self._table_sections.append((raw_key, {}))
return
key = _normalize_metric(raw_key)
value = _finite_float(raw_value)
value = finite_float(raw_value)
if not key or value is None:
return
with self._lock:
@@ -401,7 +614,7 @@ class TrainingJob:
for _name, items in sections:
table.update(items)
if steps is None:
steps = _finite_float(table.get("total_timesteps"))
steps = finite_float(table.get("total_timesteps"))
if steps is None:
# 半张表:交给调用方决定是否已确认结束(终止时按末尾兜底提交)
with self._lock:
@@ -432,7 +645,7 @@ class TrainingJob:
def _append_metric(self, point: dict[str, Any]) -> None:
clean: dict[str, Any] = {}
for key, value in point.items():
number = _finite_float(value)
number = finite_float(value)
clean[key] = number if number is not None else value
# 补齐 timesteps:progress JSON 优先,SB3 表用 total_timesteps。
if "timesteps" not in clean and "total_timesteps" in clean:
@@ -471,7 +684,7 @@ class TrainingJob:
row = {}
valid = True
for key, raw in zip(header, values):
number = _finite_float(raw)
number = finite_float(raw)
if number is None:
valid = False
break
@@ -505,17 +718,17 @@ class TrainingJob:
config = dict(self.config.__dict__)
command = list(self.command)
model_path = str(self.config.model_path)
timesteps = int(self.config.timesteps)
payload = {
"job_id": self.job_id,
"status": self.status,
"pid": _pid(self.process),
"pid": process_pid(self.process),
"created_at": self.created_at,
"started_at": self.started_at,
"finished_at": self.finished_at,
"elapsed_s": round(elapsed, 3),
"returncode": self.returncode,
"error": self.error,
"forced_kill": self.forced_kill,
"command": command,
"model": model_path,
"config": config,
@@ -529,356 +742,12 @@ class TrainingJob:
}
payload["convergence"] = summarize_convergence(metrics, monitor)
payload["process"] = describe_process(payload)
# 恢复指针在锁外做文件扫描:即使训练被强杀,这里也能给出可续训的产物
payload["recovery"] = discover_recovery(
self.config.model_path.parent, self.config.model_path)
return payload
def _series(metrics: list[dict[str, Any]], key: str) -> list[tuple[float, float]]:
points: list[tuple[float, float]] = []
for point in metrics:
value = _finite_float(point.get(key))
steps = _finite_float(point.get("timesteps")) or _finite_float(
point.get("total_timesteps"))
if value is None:
continue
points.append((steps if steps is not None else float(len(points) + 1), value))
return points
def _last_value(points: list[tuple[float, float]]) -> float | None:
return points[-1][1] if points else None
def _spread(points: list[tuple[float, float]], window: int = 10
) -> tuple[float, float] | None:
"""最近 window 个点的 (极差, 起点绝对值),用于判断"是否还在动"。"""
sample = points[-window:]
if len(sample) < 3:
return None
ys = [item[1] for item in sample]
return max(ys) - min(ys), abs(ys[0])
def _cv(points: list[tuple[float, float]], window: int = 10) -> float | None:
"""最近 window 个点的变异系数(标准差/|均值|),越小越平稳。"""
sample = [item[1] for item in points[-window:]]
if len(sample) < 3:
return None
mean = sum(sample) / len(sample)
if abs(mean) < 1e-12:
return None
var = sum((value - mean) ** 2 for value in sample) / len(sample)
return (var ** 0.5) / abs(mean)
def _slope(points: list[tuple[float, float]], window: int = 8) -> float | None:
sample = points[-window:]
if len(sample) < 3:
return None
xs = [item[0] for item in sample]
ys = [item[1] for item in sample]
n = len(xs)
mean_x = sum(xs) / n
mean_y = sum(ys) / n
denom = sum((x - mean_x) ** 2 for x in xs)
if abs(denom) < 1e-12:
return None
return sum((x - mean_x) * (y - mean_y) for x, y in zip(xs, ys)) / denom
def _rate_per_1k(reward_points: list[tuple[float, float]],
window: int = 8) -> float | None:
"""最近 window 个点的奖励增益换算成"每 1000 环境步",便于读趋势。"""
slope = _slope(reward_points, window)
if slope is None:
return None
return slope * 1000.0
def _fmt_num(value: float | None, digits: int = 4) -> str:
if value is None:
return "—"
if value != 0 and abs(value) < 1e-3:
return f"{value:.3e}"
return f"{value:.{digits}g}"
def _flatness(points: list[tuple[float, float]], window: int = 10
) -> tuple[float, float] | None:
"""窗口内"趋势位移 / 逐点噪声"比值:<1 表示看不出系统性变化。
判定收敛时只看斜率绝对值会把"很慢但持续的提升"误判为平台;这里把窗口
对半切,用两半均值之差当趋势位移,用相邻点平均绝对差当噪声尺度。返回
(比值, 噪声尺度),两者都小才是真的"又平又稳"。
"""
sample = points[-window:]
if len(sample) < 4:
return None
ys = [item[1] for item in sample]
half = len(ys) // 2
shift = abs(sum(ys[half:]) / len(ys[half:]) - sum(ys[:half]) / len(ys[:half]))
noise = sum(abs(b - a) for a, b in zip(ys, ys[1:])) / (len(ys) - 1)
if noise <= 0:
return (0.0, 0.0) if shift <= 0 else (float("inf"), 0.0)
return shift / noise, noise
def summarize_convergence(metrics: list[dict[str, Any]],
monitor: list[dict[str, Any]]) -> dict[str, Any]:
"""给前端返回可解释的收敛状态与各参数最近值。
这是**启发式判读**,不替代正式学习曲线分析:只依据最近窗口的斜率、
极差与变异系数给出"平坦/上升/未上升"的分类,并把同一组数字交给前端
展示,便于人工复核。
"""
reward_points = _series(metrics, "ep_rew_mean")
if not reward_points and monitor:
reward_points = [(float(index), float(row.get("r", 0.0)))
for index, row in enumerate(monitor)
if _finite_float(row.get("r")) is not None]
kl_points = _series(metrics, "approx_kl")
ev_points = _series(metrics, "explained_variance")
vf_points = _series(metrics, "value_loss")
pg_points = _series(metrics, "policy_gradient_loss")
entropy_points = _series(metrics, "entropy_loss")
fps_points = _series(metrics, "fps")
reward_slope = _slope(reward_points)
reward_rate = _rate_per_1k(reward_points)
spread = _spread(reward_points)
cv = _cv(reward_points)
latest_kl = _last_value(kl_points)
latest_ev = _last_value(ev_points)
latest_vf = _last_value(vf_points)
latest_pg = _last_value(pg_points)
latest_entropy = _last_value(entropy_points)
latest_fps = _last_value(fps_points)
latest_timesteps = metrics[-1].get("timesteps") if metrics else None
if latest_timesteps is None and metrics:
latest_timesteps = metrics[-1].get("total_timesteps")
latest_reward = _last_value(reward_points)
latest_len = _last_value(_series(metrics, "ep_len_mean"))
latest_updates = _last_value(_series(metrics, "n_updates"))
window = min(10, max(3, len(reward_points)))
reward_window = [round(value, 6) for _, value in reward_points[-window:]]
best_reward = max((value for _, value in reward_points), default=None)
notes: list[str] = []
if not metrics and not monitor:
status, tone = "等待首个 rollout", "gray"
notes.append("训练进程启动后先输出环境初始化与首个 rollout 的进度行。")
elif not metrics:
status, tone = "已收到回合奖励,等待进度行", "warn"
notes.append("Monitor CSV 已有回合记录,但尚未收到 --progress-json 进度行;"
"首个 rollout 结束后会出现完整指标。")
elif len(metrics) < 3:
status, tone = "预热阶段", "warn"
notes.append("已收到初始指标,需要更多 rollout 才能判断趋势。")
else:
tone, status = "ok", "收敛趋势良好"
# 平台判据(三条同时成立):对半窗口看不出位移、窗口内波动小、
# 逐点噪声本身也小。慢速但持续的提升会因第一条被排除。
flat = _flatness(reward_points)
plateau = False
if flat is not None and cv is not None and latest_reward is not None:
flat_ratio, flat_noise = flat
scale = max(abs(latest_reward), 1e-9)
plateau = (flat_ratio <= 1.0 and cv <= 0.15
and flat_noise <= 0.02 * scale
and (spread is None or spread[1] < 1e-12
or spread[0] <= 0.15 * spread[1]))
if latest_kl is not None and latest_kl > 0.05:
status, tone = "KL 偏大,策略更新可能过猛", "bad"
notes.append(f"approx_kl={latest_kl:.4f} > 0.05")
elif latest_ev is not None and latest_ev < 0.0:
status, tone = "价值函数尚未拟合", "warn"
notes.append(f"explained_variance={latest_ev:.3f} < 0")
elif reward_slope is not None and reward_slope < 0:
status, tone = "奖励未形成上升趋势", "warn"
notes.append(f"最近 {window} 点奖励斜率为负({_fmt_num(reward_slope)}/step)")
elif plateau:
status, tone = "已趋于平稳(疑似收敛)", "ok"
notes.append("窗口内对半位移小于逐点噪声,且波动很小"
f"(变异系数 {_fmt_num(cv, 3)})")
else:
notes.append(f"最近 {window} 点奖励斜率为正({_fmt_num(reward_slope)}/step)")
if latest_kl is not None and latest_kl <= 0.02:
notes.append(f"approx_kl={latest_kl:.4f} 较稳定")
if latest_ev is not None:
notes.append(f"explained_variance={latest_ev:.3f}")
if cv is not None:
notes.append(f"奖励变异系数={_fmt_num(cv, 3)}")
if latest_entropy is not None and abs(latest_entropy) < 1e-3:
notes.append("策略熵接近 0,探索可能过早停止")
return {
"status": status,
"tone": tone,
"notes": notes,
"window": window,
"latest_timesteps": latest_timesteps,
"reward_slope": reward_slope,
"reward_rate_per_1k": reward_rate,
"reward_cv": cv,
"reward_spread": spread[0] if spread else None,
"reward_window": reward_window,
"best_reward": best_reward,
"latest_reward": latest_reward,
"latest_ep_len": latest_len,
"latest_updates": latest_updates,
"latest_kl": latest_kl,
"latest_explained_variance": latest_ev,
"latest_value_loss": latest_vf,
"latest_policy_loss": latest_pg,
"latest_entropy_loss": latest_entropy,
"latest_fps": latest_fps,
"reward_points": len(reward_points),
}
# 快照状态 -> 界面展示用的进程语义(阶段名称、色调、是否终态)
_STATUS_VIEW = {
"created": ("已创建", "gray", False, "命令已构造,等待 spawn"),
"running": ("运行中", "ok", False, "子进程持有控制台输出与独立进程组"),
"stopping": ("停止中", "warn", False, "已发送 SIGTERM,等待子进程退出"),
"stopped": ("已手动停止", "warn", True, "收到停止信号后退出;脚本会在 step 边界保存模型与元数据"),
"exited": ("正常结束", "ok", True, "退出码 0:训练与评估流程走完"),
"failed": ("异常结束", "bad", True, "非零退出码或启动失败,见日志尾部"),
}
_PHASES = [
("created", "构造命令"),
("spawn", "启动子进程"),
("rollout", "采集 rollout"),
("update", "PPO 更新"),
("evaluate", "评估 / 保存"),
("done", "收敛归档"),
]
# 状态 -> 当前所处的阶段节点;节点之前为已完成,之后为未开始。
_STATUS_PHASE = {
"created": "created",
"running": "update",
"stopping": "update",
"stopped": "done",
"exited": "done",
"failed": "done",
}
def _pid(process: subprocess.Popen | None) -> int | None:
"""已退出进程的 pid 访问会抛 OSError;这里统一吞掉,只用于展示。"""
if process is None:
return None
try:
return int(process.pid)
except (OSError, ValueError):
return None
def describe_process(snapshot: dict[str, Any]) -> dict[str, Any]:
"""把 snapshot 派生为界面直接可渲染的进程对象。
包含:状态语义词、进度(步/百分比)、墙钟与剩余时间估计、最近吞吐、
进程时间线节点。数字都取自 snapshot,前端不再重复累加,避免两个刷新
频率不同的计数器互相打架。
"""
status = str(snapshot.get("status") or "created")
label, tone, terminal, hint = _STATUS_VIEW.get(
status, (status, "gray", False, ""))
config = snapshot.get("config") or {}
order = [key for key, _ in _PHASES]
current = _STATUS_PHASE.get(status, "created")
timeline = []
for key, title in _PHASES:
index = order.index(key)
if index < order.index(current):
state = "done"
elif index == order.index(current):
state = "active"
else:
state = "pending"
timeline.append({"key": key, "label": title, "state": state})
metrics = snapshot.get("metrics") or []
latest: dict[str, Any] = metrics[-1] if metrics else {}
convergence = snapshot.get("convergence") or {}
total_steps = _finite_float(config.get("timesteps")) or 0.0
# SB3 表格行可能没有 steps 字段,因此进度取"所有进度点的最大步数",
# 而不是只看最后一个点——否则末行恰好缺字段时进度会回退到 0。
done_steps = 0.0
for point in metrics:
value = (_finite_float(point.get("timesteps"))
or _finite_float(point.get("total_timesteps")))
if value is not None and value > done_steps:
done_steps = value
if done_steps == 0.0:
done_steps = _finite_float(convergence.get("latest_timesteps")) or 0.0
# init/resume(课程迁移)时 num_timesteps 从上一阶段继续,进度必须按本次
# 增量算;start_timesteps 由训练脚本的进度 JSON 提供,缺省按 0 处理。
start_steps = 0.0
for point in metrics:
value = _finite_float(point.get("start_timesteps"))
if value is not None:
start_steps = value
break
trained_steps = max(0.0, done_steps - start_steps)
percent = (100.0 * trained_steps / total_steps) if total_steps > 0 else None
if percent is not None:
percent = max(0.0, min(100.0, percent))
elapsed = float(snapshot.get("elapsed_s") or 0.0)
rate = _latest_rate(metrics)
remaining: float | None = None
eta_ts: float | None = None
if rate and rate > 0 and total_steps > 0 and trained_steps < total_steps:
remaining = (total_steps - trained_steps) / rate
eta_ts = float(snapshot.get("finished_at") or time.time()) + remaining
if not terminal and remaining is None:
cap = _finite_float(config.get("max_wall_s")) or 0.0
if cap > 0:
remaining = max(0.0, cap - elapsed)
return {
"label": label,
"tone": tone,
"terminal": terminal,
"hint": hint,
"phase": current,
"timeline": timeline,
"percent": percent,
"done_steps": done_steps,
"start_steps": start_steps,
"trained_steps": trained_steps,
"total_steps": total_steps,
"elapsed_s": round(elapsed, 3),
"rate_per_s": rate,
"remaining_s": remaining,
"eta_epoch_s": eta_ts,
"log_total": snapshot.get("log_total"),
"log_capacity": MAX_LOG_LINES,
"metric_total": snapshot.get("metric_total"),
"latest": latest,
}
def _latest_rate(metrics: list[dict[str, Any]], window: int = 6) -> float | None:
"""最近若干个**带 fps 的**进度点的平均吞吐(步/秒),用于剩余时间估计。
progress JSON 行不含 fps,只有 SB3 表格行有;若只看最后 window 个点,
恰好落在若干 JSON 行之间时会得到 None,剩余时间就会退化成墙钟上限的
粗糙估计。这里向前扫描直到拿到至少 1 个 fps(最多看 3*window 个点)。
"""
if not metrics:
return None
usable: list[float] = []
for point in reversed(metrics[-window * 3:]):
value = _finite_float(point.get("fps"))
if value and value > 0:
usable.append(value)
if len(usable) >= window:
break
if not usable:
return None
return sum(usable) / len(usable)
class TrainingUI:
"""训练前端管理器:持有唯一 TrainingJob 与 HTTP 服务器。"""
@@ -901,9 +770,9 @@ class TrainingUI:
config = TrainConfig.from_payload(payload)
with self._lock:
if self.job is not None and self.job.status in (
"created", "running", "stopping"):
"created", "running", "paused", "stopping"):
raise RuntimeError("已有训练任务在运行,请先停止")
self.job = TrainingJob(config)
self.job = TrainingJob(config, control_dir=CONTROL_DIR)
try:
self.job.start()
except Exception:
@@ -913,6 +782,14 @@ class TrainingUI:
# 与 snapshot() 同形:HTTP 层直接把返回值当响应体,不再二次拆包
return {"job": self.job.snapshot(log_tail=1)}
def pause(self) -> bool:
with self._lock:
return bool(self.job and self.job.pause())
def resume(self) -> bool:
with self._lock:
return bool(self.job and self.job.resume())
def stop(self) -> bool:
with self._lock:
return bool(self.job and self.job.stop())
@@ -938,7 +815,10 @@ class TrainingUI:
self._serving = False
def shutdown(self) -> None:
self.stop()
job = self.job
if job is not None and job.stop():
# 留出收尾落盘的时间,避免前端先退出把训练子进程留成孤儿
job.wait_for_exit(timeout=STOP_GRACE_S + 5.0)
if self._serving:
self.httpd.shutdown()
self.httpd.server_close()
@@ -948,8 +828,8 @@ class _UIHandler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
server_version = "MCM26B-Training-UI/1.0"
def log_message(self, format, *args): # noqa: A002
pass
def log_message(self, format, *args): # noqa: A002 # pylint: disable=redefined-builtin
"""覆写 stdlib 签名以吞掉访问日志;参数名沿用 ``format`` 不能改。"""
@property
def ui(self) -> TrainingUI:
@@ -999,6 +879,10 @@ class _UIHandler(BaseHTTPRequestHandler):
try:
if parts.path == "/api/train/start":
self._send_json(200, self.ui.start(payload))
elif parts.path == "/api/train/pause":
self._send_json(200, {"paused": self.ui.pause()})
elif parts.path == "/api/train/resume":
self._send_json(200, {"resumed": self.ui.resume()})
elif parts.path == "/api/train/stop":
self._send_json(200, {"stopped": self.ui.stop()})
else:
+100 -15
View File
@@ -23,6 +23,8 @@ const state = {
chartKey: "ep_rew_mean",
jobKey: null, // job_id 变化时清空本地缓存
lastLogSignature: "",
lastStatus: null, // 最近一次快照的状态:决定暂停按钮是"暂停"还是"继续"
lastJob: null, // 最近一次快照:恢复按钮需要读 job.recovery
};
// ---------------------------------------------------------------- 工具
@@ -285,9 +287,11 @@ function absorbMetrics(job) {
// ---------------------------------------------------------------- 渲染
function render(job) {
if (!job) {
state.lastJob = null;
renderIdle();
return;
}
state.lastJob = job;
if (job.job_id !== state.jobKey) {
state.jobKey = job.job_id;
state.metrics = [];
@@ -302,6 +306,7 @@ function render(job) {
renderLog(job);
renderEpisodes(job);
renderBanner(job);
renderRecovery(job);
renderJobLine(job);
}
@@ -319,6 +324,10 @@ function renderIdle() {
setText("stat-logs", "0");
$("timeline").innerHTML = "";
setText("command-preview", "");
state.lastStatus = null;
const pauseBtn = $("btn-pause");
pauseBtn.disabled = true;
pauseBtn.textContent = "暂停训练";
chip("chip-converge", "判读", "等待数据", "gray");
$("converge-notes").innerHTML = "";
$("metric-grid").innerHTML = '<p class="hint">尚未启动训练,暂无参数。</p>';
@@ -370,8 +379,14 @@ function renderProcess(job) {
renderTimeline(process);
setText("command-preview", job.command ? `$ ${job.command.join(" ")}` : "");
setFormLocked(job.status === "running" || job.status === "created" || job.status === "stopping");
$("btn-stop").disabled = !(job.status === "running" || job.status === "created");
state.lastStatus = job.status;
setFormLocked(job.status === "running" || job.status === "created"
|| job.status === "paused" || job.status === "stopping");
$("btn-stop").disabled = !(job.status === "running" || job.status === "created"
|| job.status === "paused");
const pauseBtn = $("btn-pause");
pauseBtn.disabled = !(job.status === "running" || job.status === "paused");
pauseBtn.textContent = job.status === "paused" ? "继续训练" : "暂停训练";
}
function renderTimeline(process) {
@@ -427,8 +442,12 @@ function renderBanner(job) {
}
if (job.status === "failed") {
showBanner("bad", `训练异常结束(退出码 <b>${job.returncode === null ? "?" : job.returncode}</b>):${escapeHtml(job.error || "见日志尾部")}。`);
} else if (job.status === "paused") {
showBanner("warn", "训练已暂停:进程与显存仍在,但停在 step 边界不再推进。点「继续训练」从暂停处恢复,或点「停止训练」保存当前模型后退出。");
} else if (job.status === "stopped") {
showBanner("warn", "已按请求停止训练(SIGTERM 独立进程组),脚本会在 step 边界保存当前模型与元数据。");
showBanner("warn", job.forced_kill
? "停止请求超时,已强制结束进程树;最后一段模型可能未保存,请检查 checkpoint 目录。"
: "已按请求停止训练(控制文件协作式停止),脚本已在 step 边界保存当前模型与元数据。");
} else if (job.status === "exited") {
showBanner("ok", "训练与评估流程正常结束,模型与元数据已写入输出目录。");
} else if (process.remaining_s !== null && process.remaining_s !== undefined
@@ -439,6 +458,26 @@ function renderBanner(job) {
}
}
// ---------------------------------------------------------------- 恢复
function renderRecovery(job) {
const recovery = job.recovery || {};
const box = $("recovery-box");
if (!box) return;
if (!recovery.model || recovery.reason === "none") {
box.classList.add("hidden");
return;
}
box.classList.remove("hidden");
const steps = recovery.timesteps === null || recovery.timesteps === undefined
? "步数未知" : `${fmtInt(recovery.timesteps)} 步`;
const when = recovery.updated_unix_s ? fmtClock(recovery.updated_unix_s) : "—";
setText("recovery-text",
`可恢复产物:${steps} · 来源 ${recovery.reason || "?"} · 更新 ${when} · ${recovery.model}`);
const button = $("btn-recover");
// 离线预览里的恢复信息是合成的,不允许真的去续训
button.disabled = !recovery.can_resume || Boolean(state.preview);
}
// ---------------------------------------------------------------- 收敛
function renderConvergence(job) {
const convergence = job.convergence || {};
@@ -679,6 +718,33 @@ function renderEpisodes(job) {
}
// ---------------------------------------------------------------- 交互
async function submitPayload(payload) {
const err = $("form-error");
err.textContent = "";
if (state.saving) return false; // 训练中表单锁定,续训按钮同样不许重复提交
if (payload.resume && !payload.model) {
err.textContent = "续训需要填写模型输出路径";
return false;
}
const res = await fetch("/api/train/start", {
method: "POST", headers: { "Content-Type": "application/json" },
body: JSON.stringify(payload),
});
const body = await res.json().catch(() => ({}));
if (!res.ok) {
err.textContent = body.error || `启动失败(HTTP ${res.status})`;
return false;
}
state.preview = false;
state.jobKey = null;
state.metrics = [];
state.metricCache = [];
state.metricNext = 0;
state.metricTotal = 0;
await poll();
return true;
}
$("config-form").addEventListener("submit", async (event) => {
event.preventDefault();
if (state.saving) return;
@@ -691,24 +757,43 @@ $("config-form").addEventListener("submit", async (event) => {
for (const [key, value] of Object.entries(values)) {
if (value !== null) payload[key] = value;
}
if (payload.resume && !payload.model) { err.textContent = "续训需要填写模型输出路径"; return; }
const res = await fetch("/api/train/start", {
method: "POST", headers: { "Content-Type": "application/json" },
body: JSON.stringify(payload),
await submitPayload(payload);
});
$("btn-recover").addEventListener("click", async () => {
const recovery = (state.lastJob && state.lastJob.recovery) || {};
if (!recovery.can_resume || !recovery.model) return;
if (!confirm(`用最近的可恢复产物续训?\n${recovery.model}\n`
+ "当前表单里的 timesteps 会作为**追加**步数。")) return;
// 同步回表单,用户随后可微调步数再手动点「开始训练」
const modelField = $("cfg-model");
if (modelField) modelField.value = recovery.model;
const resumeField = $("cfg-resume");
if (resumeField) resumeField.checked = true;
const values = collectValues();
values.model = recovery.model;
values.resume = true;
values.force = false; // --resume 与 --force 互斥
await submitPayload(values);
});
$("btn-pause").addEventListener("click", async () => {
const resuming = state.lastStatus === "paused";
const res = await fetch(resuming ? "/api/train/resume" : "/api/train/pause", {
method: "POST", headers: { "Content-Type": "application/json" }, body: "{}",
});
const body = await res.json().catch(() => ({}));
if (!res.ok) { err.textContent = body.error || `启动失败(HTTP ${res.status})`; return; }
state.preview = false;
state.jobKey = null;
state.metrics = [];
state.metricCache = [];
state.metricNext = 0;
state.metricTotal = 0;
if (!res.ok) {
showBanner("bad", `${resuming ? "继续" : "暂停"}失败:${escapeHtml(body.error || res.status)}`);
} else if (!resuming && body.paused === false) {
showBanner("warn", "当前没有可暂停的训练进程(可能已结束)。");
}
await poll();
});
$("btn-stop").addEventListener("click", async () => {
if (!confirm("确认停止当前训练?\n将向独立进程组发送 SIGTERM,脚本会先保存当前模型与元数据再退出。")) return;
if (!confirm("确认停止当前训练?\n将通过控制文件请求优雅停止,脚本先保存当前模型与元数据再退出;"
+ "若超时未响应则强制结束进程树。")) return;
const res = await fetch("/api/train/stop", {
method: "POST", headers: { "Content-Type": "application/json" }, body: "{}",
});
+6
View File
@@ -30,6 +30,7 @@
</select>
</label>
<button id="btn-preview" type="button" title="渲染示例快照,不启动任何训练进程">离线预览</button>
<button id="btn-pause" type="button" disabled>暂停训练</button>
<button id="btn-stop" class="danger" type="button" disabled>停止训练</button>
</div>
</header>
@@ -37,6 +38,11 @@
<main>
<div id="banner" class="banner hidden"></div>
<div id="recovery-box" class="recovery hidden">
<span class="hint" id="recovery-text">—</span>
<button id="btn-recover" type="button" title="把最近一次完整落盘的产物作为 --resume 目标继续训练">用最近检查点续训</button>
</div>
<!-- =============================================== 1. 执行进度与进程 -->
<section class="panel" id="panel-process">
<div class="panel-head">
+9
View File
@@ -64,6 +64,15 @@ button.danger:hover:enabled { background: rgba(248,113,113,.12); }
.banner.warn { border-color: var(--warn); border-left-color: var(--warn); background: rgba(245,158,11,.08); }
.banner b { font-family: var(--mono); }
/* 恢复条:显示最近一份完整落盘的产物,并提供一键续训 */
.recovery {
display: flex; align-items: center; justify-content: space-between; gap: 12px;
border: 1px solid var(--cyan, #22d3ee); border-left: 4px solid var(--cyan, #22d3ee);
background: rgba(34,211,238,.07); border-radius: 8px;
padding: 8px 12px; margin: 10px 0 0; font-size: 13px;
}
.recovery .hint { overflow-wrap: anywhere; }
/* ------------------------------------------------------------ 面板 */
.panel {
border: 1px solid var(--line); border-radius: 12px; background: var(--panel);