holy fuking shit
This commit is contained in:
@@ -126,10 +126,10 @@ class CandidateRegion:
|
||||
@classmethod
|
||||
def initial(cls, station: Vec2, config: ProblemConfig) -> "CandidateRegion":
|
||||
"""首个 direction 后的基础集:Ω ∩ B(station, r_max) 的保守外包。"""
|
||||
field, sides = config.field, config.search.polygon_sides
|
||||
slack = field.omega_radius_m * (1 - math.cos(math.pi / sides))
|
||||
region = cls.disk((0.0, 0.0), field.omega_radius_m, sides, slack)
|
||||
region.clip_disk(station, field.r_eff_max_m, sides)
|
||||
field_cfg, sides = config.field, config.search.polygon_sides
|
||||
slack = field_cfg.omega_radius_m * (1 - math.cos(math.pi / sides))
|
||||
region = cls.disk((0.0, 0.0), field_cfg.omega_radius_m, sides, slack)
|
||||
region.clip_disk(station, field_cfg.r_eff_max_m, sides)
|
||||
return region
|
||||
|
||||
# ------------------------------------------------------------ 收缩
|
||||
@@ -198,8 +198,7 @@ class CandidateRegion:
|
||||
if len(self.polygon) == 2:
|
||||
p, q = self.polygon
|
||||
return _point_segment_distance(point, p, q) <= tol_m + self.slack_m
|
||||
for index in range(len(self.polygon)):
|
||||
p = self.polygon[index]
|
||||
for index, p in enumerate(self.polygon):
|
||||
q = self.polygon[(index + 1) % len(self.polygon)]
|
||||
if (q[0] - p[0]) * (point[1] - p[1]) - (q[1] - p[1]) * (point[0] - p[0]) \
|
||||
< -tol_m - self.slack_m:
|
||||
|
||||
@@ -294,6 +294,13 @@ def _run(args) -> int:
|
||||
if judge is not None:
|
||||
_settle_local_judge(judge)
|
||||
summary["robot_id"] = args.robot_id
|
||||
return _finalize_run(args, summary, strategy, truth=truth, judge=judge,
|
||||
judge_session=judge_session, real_budget_s=real_budget_s)
|
||||
|
||||
|
||||
def _finalize_run(args, summary, strategy, *, truth, judge, judge_session,
|
||||
real_budget_s) -> int:
|
||||
"""补齐 summary 元信息、核对真值并输出;返回进程退出码。"""
|
||||
if args.problem == 4:
|
||||
summary["strategy"] = "rl_q4" if strategy == "rl" else strategy
|
||||
else:
|
||||
|
||||
@@ -32,7 +32,7 @@ class CoverageLedger:
|
||||
"""逐频道覆盖账本:(频道, 网格点) -> 是否已测。"""
|
||||
|
||||
config: ProblemConfig = field(default_factory=ProblemConfig)
|
||||
points: list[Vec2] = field(default_factory=lambda: coverage_points())
|
||||
points: list[Vec2] = field(default_factory=coverage_points)
|
||||
measured: set = field(default_factory=set) # {(channel, point_index)}
|
||||
skipped: dict = field(default_factory=dict) # channel -> 原因
|
||||
|
||||
|
||||
@@ -23,8 +23,10 @@ def main(argv=None):
|
||||
measurements = [replace(m, half_angle_deg=m.half_angle_deg+args.rounding_margin_deg)
|
||||
for m in measurements]
|
||||
result = solve(measurements)
|
||||
result["error_policy"] = {"additional_half_angle_deg": args.rounding_margin_deg,
|
||||
"effective_half_angles_deg": [m.half_angle_deg for m in measurements]}
|
||||
result["error_policy"] = {
|
||||
"additional_half_angle_deg": args.rounding_margin_deg,
|
||||
"effective_half_angles_deg": [m.half_angle_deg for m in measurements],
|
||||
}
|
||||
except (ValueError, TypeError, KeyError, OSError) as exc:
|
||||
parser.error(str(exc))
|
||||
text = json.dumps(result, ensure_ascii=False, indent=2, allow_nan=False)
|
||||
|
||||
@@ -133,6 +133,13 @@ def _enclosing_circle_reference(points: list[Point], eps: float = EPS):
|
||||
return best_c, best_r
|
||||
|
||||
|
||||
def _circle_covers(circle, point: Point, tolerance: float) -> bool:
|
||||
"""Whether the circle already covers point; None means no circle yet."""
|
||||
if circle is None:
|
||||
return False
|
||||
return distance(circle[0], point) <= circle[1] + tolerance
|
||||
|
||||
|
||||
def smallest_enclosing_circle(points: list[Point], eps: float = EPS):
|
||||
"""Welzl's randomized incremental algorithm, using an explicit call stack.
|
||||
|
||||
@@ -169,8 +176,9 @@ def smallest_enclosing_circle(points: list[Point], eps: float = EPS):
|
||||
elif not returning:
|
||||
stack.append((n, boundary, True))
|
||||
stack.append((n-1, boundary, False))
|
||||
elif circle is None or distance(circle[0], local[n-1]) > circle[1] + tolerance:
|
||||
elif not _circle_covers(circle, local[n-1], tolerance):
|
||||
stack.append((n-1, boundary + (local[n-1],), False))
|
||||
assert circle is not None
|
||||
center = (origin[0]+circle[0][0]*scale, origin[1]+circle[0][1]*scale)
|
||||
# Recompute after transforming back: returned circle contains all input points
|
||||
# to distance() precision, including any roundoff from the large translation.
|
||||
|
||||
@@ -29,7 +29,7 @@ class LocalSimTransport(Transport):
|
||||
def post(self, url, body, headers, timeout):
|
||||
return self.request("POST", url, body, headers, timeout)
|
||||
|
||||
def request(self, method, url, body, headers, timeout=None):
|
||||
def request(self, method, url, body, headers, _timeout=None):
|
||||
"""验证原始请求边界;内存回环不模拟 TCP、UI 或连接限流。"""
|
||||
target = urlsplit(url)
|
||||
raw_target = target.path + (f"?{target.query}" if target.query else "")
|
||||
@@ -57,5 +57,6 @@ class ScriptedFaultTransport(Transport):
|
||||
self.inner.post(url, body, headers, timeout) # 服务器已受理
|
||||
raise TransportError("injected: read timeout after server applied")
|
||||
if isinstance(fault, tuple) and fault and fault[0] == "status":
|
||||
return TransportResponse(int(fault[1]), b'{"accepted": false, "real_timestamp_ms": 0, "virtual_time_s": 0}')
|
||||
body = b'{"accepted": false, "real_timestamp_ms": 0, "virtual_time_s": 0}'
|
||||
return TransportResponse(int(fault[1]), body)
|
||||
return self.inner.post(url, body, headers, timeout)
|
||||
|
||||
@@ -17,7 +17,7 @@ import math
|
||||
import numpy as np
|
||||
|
||||
from ..config import Vec2
|
||||
from .constants import (LOW_MAX_STEPS, LOW_NUM_ACTIONS, LOW_OBS_DIM,
|
||||
from .constants import (LOW_MAX_STEPS, LOW_NUM_ACTIONS,
|
||||
LOW_REACH_M, LOW_STEP_M, POS_SCALE_M)
|
||||
|
||||
__all__ = ["build_low_observation", "LowLevelExecutor", "action_bearing",
|
||||
|
||||
@@ -144,7 +144,6 @@ class HierarchicalEnv(gym.Env):
|
||||
problem=self.problem)
|
||||
|
||||
def _training_spec(self):
|
||||
from ..harness import referee_spec
|
||||
from ..hrl.curriculum import training_spec
|
||||
return training_spec(self.train_config, self.stage,
|
||||
max_real_duration_s=self.real_budget_s)
|
||||
|
||||
@@ -19,7 +19,7 @@ from gymnasium import spaces
|
||||
|
||||
from ..config import ProblemConfig
|
||||
from .constants import (LOW_MAX_STEPS, LOW_NUM_ACTIONS, LOW_OBS_DIM,
|
||||
LOW_REACH_M, LOW_STEP_M, POS_SCALE_M)
|
||||
LOW_REACH_M, LOW_STEP_M)
|
||||
from .executor import (action_bearing, build_low_observation,
|
||||
navigation_transition)
|
||||
|
||||
|
||||
@@ -48,7 +48,7 @@ def _load_maskable(model_path: Path, device: str, *,
|
||||
if meta_path.exists() else {}
|
||||
# 高层模型必须携带完整语义元数据;缺字段不能默认兼容,防止旧槽位/
|
||||
# 旧奖励语义模型被静默加载。低层导航模型只校验基础特征版本。
|
||||
strict = (act_dim == N_CANDIDATES)
|
||||
strict = act_dim == N_CANDIDATES
|
||||
if strict:
|
||||
required = ("feature_version", "action_layout_version",
|
||||
"reward_version", "algorithm", "problem",
|
||||
|
||||
@@ -29,7 +29,7 @@ from typing import NamedTuple
|
||||
import numpy as np
|
||||
|
||||
from ..belief import ChannelStatus
|
||||
from ..config import ProblemConfig, Vec2
|
||||
from ..config import Vec2
|
||||
from ..protocol.rules import CLEAR_SUCCESS, DIRECTION, NEAR, NO_SIGNAL
|
||||
from ..protocol.timing import move_time
|
||||
from .constants import (CHANNEL_COUNT, CLEAR_OFFSET, COST_SCALE_S,
|
||||
@@ -84,6 +84,8 @@ class HRLControllerMixin:
|
||||
"""
|
||||
|
||||
problem: int
|
||||
# 由宿主 BayesLoop.__init__ 初始化;此处声明让 pylint 可见该诊断计数。
|
||||
no_shrink_measures: int = 0
|
||||
|
||||
# ------------------------------------------------------------ 候选构建
|
||||
def build_candidates(self) -> list[Candidate]:
|
||||
@@ -220,7 +222,7 @@ class HRLControllerMixin:
|
||||
def hrl_observation(self, candidates: list[Candidate]) -> np.ndarray:
|
||||
"""构造高层观测(shape=(HIGH_OBS_DIM,),训练/推理逐位一致)。"""
|
||||
config = self.config
|
||||
field = config.field
|
||||
field_cfg = config.field
|
||||
position = self.client.state.position
|
||||
belief = self.belief
|
||||
|
||||
@@ -232,24 +234,24 @@ class HRLControllerMixin:
|
||||
glob = [
|
||||
position[0] / POS_SCALE_M,
|
||||
position[1] / POS_SCALE_M,
|
||||
min(self.client.state.virtual_time_s, field.max_virtual_duration_s)
|
||||
/ field.max_virtual_duration_s,
|
||||
min(self.client.state.virtual_time_s, field_cfg.max_virtual_duration_s)
|
||||
/ field_cfg.max_virtual_duration_s,
|
||||
cleared / 16.0,
|
||||
accounted / CHANNEL_COUNT,
|
||||
(self.client.state.channel - field.channel_min)
|
||||
/ max(field.channel_max - field.channel_min, 1),
|
||||
(self.client.state.channel - field_cfg.channel_min)
|
||||
/ max(field_cfg.channel_max - field_cfg.channel_min, 1),
|
||||
float(self.problem == 4),
|
||||
detected_count / 16.0,
|
||||
]
|
||||
assert len(glob) == HIGH_GLOBAL_DIM
|
||||
|
||||
pending_station = np.zeros(CHANNEL_COUNT + 1, dtype=bool)
|
||||
for index, channels in getattr(self, "_station_pending_map", {}).items():
|
||||
for channels in getattr(self, "_station_pending_map", {}).values():
|
||||
for channel in channels:
|
||||
pending_station[channel] = True
|
||||
|
||||
channel_features: list[float] = []
|
||||
for channel in range(field.channel_min, field.channel_max + 1):
|
||||
for channel in range(field_cfg.channel_min, field_cfg.channel_max + 1):
|
||||
ch = belief.channels[channel]
|
||||
status6 = [0.0] * 6
|
||||
status6[self._hrl_channel_status_index(channel)] = 1.0
|
||||
@@ -363,7 +365,7 @@ class HRLControllerMixin:
|
||||
outcome = StepOutcome(action=action, kind=cand.kind,
|
||||
channel=cand.channel,
|
||||
position_reached=tuple(self.client.state.position))
|
||||
outcome.delta_virtual_s = (self.client.state.virtual_time_s - old_virtual)
|
||||
outcome.delta_virtual_s = self.client.state.virtual_time_s - old_virtual
|
||||
new_detections: list[int] = []
|
||||
info_gains: dict[int, float] = {}
|
||||
redundant = False
|
||||
|
||||
@@ -237,10 +237,11 @@ class ChannelParticles:
|
||||
base = i * self.width
|
||||
for j in range(j0, min(j0 + _COARSE_FACTOR, self.width)):
|
||||
index = base + j
|
||||
if index in self.alive:
|
||||
dist = math.dist(self.cell_center(index), point)
|
||||
if dist <= limit_m and (best is None or dist < best):
|
||||
best = dist
|
||||
if index not in self.alive:
|
||||
continue
|
||||
dist = math.dist(self.cell_center(index), point)
|
||||
if dist <= limit_m and (best is None or dist < best):
|
||||
best = dist
|
||||
return best
|
||||
|
||||
def guaranteed_no_signal(self, station: Vec2) -> bool:
|
||||
|
||||
@@ -621,7 +621,8 @@ class DirectionalExtension(OmnidirectionalLoop):
|
||||
self.ledger.mark_measured(channel, index)
|
||||
self.belief.update_measure(channel, point, outcome.result, outcome.svd_deg)
|
||||
if outcome.result == NEAR:
|
||||
cert = clear_certificate(self.belief.candidate_set(channel), channel, self.config)
|
||||
cert = clear_certificate(
|
||||
self.belief.candidate_set(channel), channel, self.config)
|
||||
if cert.certified:
|
||||
self._attempt_clear(channel, point, cert)
|
||||
if self._count_limit_reached():
|
||||
@@ -629,7 +630,8 @@ class DirectionalExtension(OmnidirectionalLoop):
|
||||
queue = self.belief.channels_by_status(ChannelStatus.DETECTED)
|
||||
while queue and self.stop_reason is None:
|
||||
channel = min(queue, key=lambda c: (
|
||||
math.dist(self.client.state.position, self.belief.candidate_set(c).mec()[0]), c))
|
||||
math.dist(self.client.state.position,
|
||||
self.belief.candidate_set(c).mec()[0]), c))
|
||||
queue.remove(channel)
|
||||
self._localize_and_clear(channel)
|
||||
if self._count_limit_reached():
|
||||
@@ -672,8 +674,7 @@ class DirectionalExtension(OmnidirectionalLoop):
|
||||
self.directional_observed.append(channel)
|
||||
self.blocked_probes.setdefault(channel, []).append(point)
|
||||
continue # 盘外无信号也可能是距离不足,不据此断言定向
|
||||
else:
|
||||
self.blocked_probes.pop(channel, None)
|
||||
self.blocked_probes.pop(channel, None)
|
||||
steps += 1
|
||||
if self.belief.channels[channel].status is ChannelStatus.CLEARED:
|
||||
return True
|
||||
|
||||
@@ -109,13 +109,15 @@ class BayesLoop(OmnidirectionalLoop):
|
||||
if cert is not None and cert.certified:
|
||||
continue
|
||||
region = self.belief.candidate_set(channel)
|
||||
if max(math.dist(point, v) for v in region.polygon) > self.config.field.r_eff_min_m - 1e-3:
|
||||
farthest = max(math.dist(point, v) for v in region.polygon)
|
||||
if farthest > self.config.field.r_eff_min_m - 1e-3:
|
||||
continue
|
||||
radius = region.mec()[1]
|
||||
vertices = region.polygon
|
||||
stride = max(1, len(vertices) // self.config.bayes.mc_representatives)
|
||||
radii = []
|
||||
delta = self.config.field.bearing_error_half_deg + self.config.search.bearing_rounding_margin_deg
|
||||
delta = (self.config.field.bearing_error_half_deg
|
||||
+ self.config.search.bearing_rounding_margin_deg)
|
||||
for target in vertices[::stride]:
|
||||
theta = math.degrees(math.atan2(target[1]-point[1], target[0]-point[0]))
|
||||
probe = CandidateRegion(list(vertices), region.slack_m)
|
||||
@@ -146,7 +148,7 @@ class BayesLoop(OmnidirectionalLoop):
|
||||
def _skip_station_channel(self, channel: int, index: int, point: Vec2) -> bool:
|
||||
"""跳测规则(按 pf.version 缓存;幸存集单调收缩,判定单调)。"""
|
||||
pf = self.particles[channel]
|
||||
if not len(pf):
|
||||
if not pf:
|
||||
return True
|
||||
cached = self._skip_cache.get((channel, index))
|
||||
if cached is not None and cached[0] == pf.version:
|
||||
@@ -219,7 +221,9 @@ class BayesLoop(OmnidirectionalLoop):
|
||||
total += self.config.cost.switch_s * (channel != task.channel)
|
||||
channel = task.channel
|
||||
elif task.kind == "station" and task.station_channels:
|
||||
pending = sorted(task.station_channels, key=lambda c: (c != channel, c))
|
||||
pending = sorted(
|
||||
task.station_channels,
|
||||
key=lambda c, channel=channel: (c != channel, c))
|
||||
total += self.config.cost.measure_s * len(pending)
|
||||
total += self.config.cost.switch_s * (len(pending) - (channel in pending))
|
||||
channel = pending[-1]
|
||||
@@ -328,7 +332,7 @@ class BayesLoop(OmnidirectionalLoop):
|
||||
return # 已证毕:清除/缺席后的 no_signal 与距离无关,不进 PF
|
||||
pf = self.particles[channel]
|
||||
pf.apply_measure(position, outcome.result, outcome.svd_deg)
|
||||
if not len(pf):
|
||||
if not pf:
|
||||
if belief.status is ChannelStatus.UNKNOWN:
|
||||
self.belief.mark_absent(channel)
|
||||
if channel not in self.pf_absent_channels:
|
||||
@@ -350,7 +354,8 @@ class BayesLoop(OmnidirectionalLoop):
|
||||
limit = (self.config.field.clear_radius_m - pf.margin_m
|
||||
- self.config.search.clear_margin_m)
|
||||
if radius <= limit:
|
||||
return self._near_clear_target(ClearCertificate(channel, center, radius, limit, True))
|
||||
return self._near_clear_target(
|
||||
ClearCertificate(channel, center, radius, limit, True))
|
||||
return cert
|
||||
|
||||
def _near_clear_target(self, cert: ClearCertificate) -> ClearCertificate:
|
||||
@@ -368,7 +373,7 @@ class BayesLoop(OmnidirectionalLoop):
|
||||
"""定位测量点:期望 log 幸存占比 + λ·移动时间 最小的候选。"""
|
||||
bayes, search = self.config.bayes, self.config.search
|
||||
pf = self.particles[channel]
|
||||
if not len(pf):
|
||||
if not pf:
|
||||
return None
|
||||
region = self.belief.candidate_set(channel)
|
||||
center, radius = pf.mec()
|
||||
|
||||
@@ -16,8 +16,6 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from ..belief import ChannelStatus
|
||||
from ..config import ProblemConfig, Vec2
|
||||
from ..coverage import CoverageLedger, coverage_points
|
||||
@@ -116,7 +114,7 @@ class HRLLoop(HRLControllerMixin, BayesLoop):
|
||||
if belief.status not in (ChannelStatus.CLEARED, ChannelStatus.ABSENT):
|
||||
if not (self.problem == 4 and outcome.result == NO_SIGNAL):
|
||||
pf.apply_measure(position, outcome.result, outcome.svd_deg)
|
||||
if not len(pf):
|
||||
if not pf:
|
||||
if belief.status is ChannelStatus.UNKNOWN:
|
||||
self.belief.mark_absent(channel)
|
||||
if channel not in self.pf_absent_channels:
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
"""跨平台子进程控制:启动隔离、协作式暂停/停止、必要时强杀整棵进程树。
|
||||
|
||||
训练前端要在 POSIX 与 Windows 上表现一致,而两边的进程模型差别正好卡在
|
||||
「停止训练」这条路径上:
|
||||
|
||||
- ``os.killpg`` / ``os.getpgid`` **只存在于 POSIX**;Windows 上访问它们会抛
|
||||
``AttributeError``。旧实现只捕获 ``ProcessLookupError`` / ``PermissionError``,
|
||||
于是 ``POST /api/train/stop`` 在 Windows 上直接 500,训练进程照跑——表现就是
|
||||
「点停止没反应」。
|
||||
- ``subprocess.Popen(start_new_session=True)`` 在 Windows 上被**静默忽略**:
|
||||
CPython 的 Windows ``_execute_child`` 形参名就是 ``unused_start_new_session``,
|
||||
要隔离进程组只能传 ``CREATE_NEW_PROCESS_GROUP``。
|
||||
- POSIX 的 SIGTERM 在 Windows 上经 ``os.kill`` 会退化成 ``TerminateProcess``:
|
||||
硬杀,训练脚本里"收到信号后保存模型"的处理器根本不会执行。
|
||||
|
||||
所以「优雅暂停 / 优雅停止」不依赖信号量语义,而是走一个**控制文件**:前端原子
|
||||
写 JSON,训练脚本每步轮询。两端都只是普通文件 IO,Windows 与 POSIX 行为一致;
|
||||
信号与强杀退居兜底,只在训练进程不响应控制文件时使用。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
__all__ = [
|
||||
"IS_WINDOWS", "ControlState", "spawn_kwargs", "write_control",
|
||||
"read_control", "write_json_atomic", "terminate_process_tree",
|
||||
]
|
||||
|
||||
# 运行期常量而非"导入期分支":调用方(含测试)可以 monkeypatch 它来模拟 Windows
|
||||
IS_WINDOWS = os.name == "nt"
|
||||
|
||||
# 只有 Windows 定义 CREATE_NEW_PROCESS_GROUP;POSIX 上取 0 表示不使用 creationflags
|
||||
_CREATE_NEW_PROCESS_GROUP = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)
|
||||
|
||||
# 发出「停止」到强杀之间留给训练脚本收尾(保存模型 + 写元数据)的秒数
|
||||
DEFAULT_GRACE_S = 15.0
|
||||
|
||||
|
||||
def spawn_kwargs() -> dict[str, Any]:
|
||||
"""返回 ``Popen`` 的平台相关参数:让训练进程独立于前端进程组。
|
||||
|
||||
POSIX 用 ``setsid``(``start_new_session``);Windows 上用
|
||||
``CREATE_NEW_PROCESS_GROUP``,否则前端所在控制台的 Ctrl+C 会连带打到训练进程。
|
||||
"""
|
||||
if IS_WINDOWS:
|
||||
if _CREATE_NEW_PROCESS_GROUP:
|
||||
return {"creationflags": _CREATE_NEW_PROCESS_GROUP}
|
||||
return {}
|
||||
return {"start_new_session": True}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ControlState:
|
||||
"""控制文件的一次快照;缺失或损坏时字段全为默认(即"无指令")。"""
|
||||
|
||||
paused: bool = False
|
||||
stop: bool = False
|
||||
seq: int = 0
|
||||
ts: float = 0.0
|
||||
|
||||
|
||||
def write_control(path: Path, *, paused: bool = False, stop: bool = False,
|
||||
seq: int | None = None) -> ControlState:
|
||||
"""原子写控制文件(同目录临时文件 + ``os.replace``)。
|
||||
|
||||
训练侧随时可能来读,必须避免读到写了一半的 JSON。``os.replace`` 在
|
||||
Windows 与 POSIX 上都是原子的(同目录、同卷)。
|
||||
"""
|
||||
path = Path(path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
state = ControlState(
|
||||
paused=bool(paused),
|
||||
stop=bool(stop),
|
||||
seq=int(seq) if seq is not None else time.time_ns(),
|
||||
ts=time.time(),
|
||||
)
|
||||
payload = {"paused": state.paused, "stop": state.stop,
|
||||
"seq": state.seq, "ts": state.ts}
|
||||
tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp")
|
||||
tmp.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
|
||||
os.replace(tmp, path)
|
||||
return state
|
||||
|
||||
|
||||
def read_control(path: Path) -> ControlState:
|
||||
"""读控制文件;缺失 / 半截 / 损坏一律当作"无指令",绝不让训练崩在控制面上。"""
|
||||
try:
|
||||
raw = Path(path).read_text(encoding="utf-8")
|
||||
payload = json.loads(raw)
|
||||
except (OSError, ValueError, UnicodeDecodeError):
|
||||
return ControlState()
|
||||
if not isinstance(payload, dict):
|
||||
return ControlState()
|
||||
return ControlState(
|
||||
paused=bool(payload.get("paused", False)),
|
||||
stop=bool(payload.get("stop", False)),
|
||||
seq=int(payload.get("seq") or 0),
|
||||
ts=float(payload.get("ts") or 0.0),
|
||||
)
|
||||
|
||||
|
||||
def write_json_atomic(path: Path, payload: Any) -> None:
|
||||
"""原子写 JSON 文件(同目录临时文件 + ``os.replace``)。
|
||||
|
||||
用于"急停恢复"相关的标记文件(``resume.json`` / ``forced_stop.json`` 等):
|
||||
读取方随时可能来读,必须避免看到写了一半的内容。
|
||||
"""
|
||||
path = Path(path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp")
|
||||
tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8")
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
def terminate_process_tree(process: subprocess.Popen | None, *,
|
||||
grace_s: float = DEFAULT_GRACE_S,
|
||||
poll_s: float = 0.2) -> bool:
|
||||
"""等待 ``grace_s`` 让协作式停止生效,超时则强杀整棵进程树。
|
||||
|
||||
返回**是否动用了强杀**:``False`` 表示进程在宽限期内自行退出(模型有机会
|
||||
落盘),``True`` 表示进程被强杀、最后一段产物可能缺失。调用方应先通过控制
|
||||
文件请求停止,再调用本函数;本函数不负责写控制文件。
|
||||
"""
|
||||
if process is None:
|
||||
return False
|
||||
if process.poll() is not None:
|
||||
return False
|
||||
deadline = time.monotonic() + max(0.0, grace_s)
|
||||
while time.monotonic() < deadline:
|
||||
if process.poll() is not None:
|
||||
return False
|
||||
time.sleep(min(poll_s, max(0.0, deadline - time.monotonic())))
|
||||
if process.poll() is not None:
|
||||
return False
|
||||
_hard_kill_tree(process)
|
||||
try:
|
||||
process.wait(timeout=10.0)
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
return True
|
||||
|
||||
|
||||
def _hard_kill_tree(process: subprocess.Popen) -> None:
|
||||
"""强杀进程树:POSIX 走进程组 SIGKILL,Windows 走 ``taskkill /T``。
|
||||
|
||||
这里所有 POSIX 专属调用都走 ``getattr`` 探测,Windows 上拿不到分支就退回
|
||||
``Popen.kill()``——不会再出现 ``module 'os' has no attribute 'killpg'``。
|
||||
"""
|
||||
if IS_WINDOWS:
|
||||
try:
|
||||
subprocess.run(["taskkill", "/F", "/T", "/PID", str(process.pid)],
|
||||
capture_output=True, check=False)
|
||||
return
|
||||
except OSError:
|
||||
process.kill()
|
||||
return
|
||||
killpg = getattr(os, "killpg", None)
|
||||
getpgid = getattr(os, "getpgid", None)
|
||||
sigkill = getattr(signal, "SIGKILL", None)
|
||||
if killpg is not None and getpgid is not None and sigkill is not None:
|
||||
try:
|
||||
killpg(getpgid(process.pid), sigkill)
|
||||
return
|
||||
except (ProcessLookupError, PermissionError, OSError):
|
||||
pass
|
||||
process.kill()
|
||||
@@ -13,6 +13,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from time import monotonic, sleep
|
||||
@@ -91,7 +92,7 @@ def _validate_position(position: dict, limit: float) -> None:
|
||||
value = position[key]
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
raise ValueError(f"position.{key} 必须是数值")
|
||||
if value != value or value in (float("inf"), float("-inf")):
|
||||
if not math.isfinite(value):
|
||||
raise ValueError(f"position.{key} 必须是有限数值")
|
||||
if abs(value) > limit:
|
||||
raise ValueError(f"position.{key} 绝对值不得超过 {limit} 米")
|
||||
@@ -234,7 +235,7 @@ class SimulatorClient:
|
||||
f"剩余现实时间 {self.real_remaining_s:.1f}s 不足以预留 {margin:.1f}s 退出余量")
|
||||
|
||||
# ---------------------------------------------------------------- 内部机制
|
||||
def _payload(self, path: str, *, position: Vec2 | None = None,
|
||||
def _payload(self, _path: str, *, position: Vec2 | None = None,
|
||||
channel: int | None = None) -> dict:
|
||||
payload = {"arena_id": self.session.arena_id,
|
||||
"robot_id": self.robot_id,
|
||||
|
||||
+14
-12
@@ -27,7 +27,7 @@ from ..config import ProblemConfig
|
||||
from ..planner import clear_certificate, fallback_clear_cells, safe_disk_radius
|
||||
from ..protocol.client import SimulatorClient
|
||||
from ..protocol.log import SessionLog
|
||||
from ..protocol.rules import CLEAR_SUCCESS, DIRECTION, NEAR, NO_SIGNAL
|
||||
from ..protocol.rules import CLEAR_SUCCESS, DIRECTION, NO_SIGNAL
|
||||
from .features import NUM_ACTIONS, OBS_DIM, action_mask, build_observation, candidate_actions
|
||||
|
||||
|
||||
@@ -65,9 +65,9 @@ class LocalizationEnv(gym.Env):
|
||||
from referee import LocalSimulator as _RefereeLocalSimulator
|
||||
from referee import Source
|
||||
from ..harness import LocalSimTransport, referee_spec
|
||||
self._Source = Source
|
||||
self._LocalSimulator = _RefereeLocalSimulator
|
||||
self._LocalSimTransport = LocalSimTransport
|
||||
self._source_cls = Source
|
||||
self._local_simulator_cls = _RefereeLocalSimulator
|
||||
self._local_sim_transport_cls = LocalSimTransport
|
||||
self._referee_spec = referee_spec
|
||||
|
||||
self._scene_rng = Random()
|
||||
@@ -106,23 +106,22 @@ class LocalizationEnv(gym.Env):
|
||||
# 首次检测点:距离 [300, min(1450, r_eff-20)] 内随机,定向源取朝向
|
||||
# ±89.9°(接近但不跨越90°边界,保证direction)。机器狗可到Ω外 [题面]。
|
||||
max_distance = max(310.0, min(1450.0, r_eff - 20.0))
|
||||
if max_distance < 310.0:
|
||||
max_distance = 310.0
|
||||
max_distance = max(max_distance, 310.0)
|
||||
probe_distance = rng.uniform(300.0, max_distance)
|
||||
probe_bearing = (heading + rng.uniform(-89.9, 89.9)
|
||||
if directional else rng.uniform(0.0, 360.0))
|
||||
probe_radians = math.radians(probe_bearing)
|
||||
probe = (x + probe_distance * math.cos(probe_radians),
|
||||
y + probe_distance * math.sin(probe_radians))
|
||||
source = self._Source(channel=channel, x=x, y=y, r_eff_m=r_eff,
|
||||
directional=directional, heading_deg=heading)
|
||||
source = self._source_cls(channel=channel, x=x, y=y, r_eff_m=r_eff,
|
||||
directional=directional, heading_deg=heading)
|
||||
return source, probe
|
||||
|
||||
def _make_episode(self) -> bool:
|
||||
"""重建模拟器并完成首次测向;返回 False 表示连续未得到 direction。"""
|
||||
for _ in range(20):
|
||||
self._source, probe = self._make_source_and_probe()
|
||||
self._simulator = self._LocalSimulator(
|
||||
self._simulator = self._local_simulator_cls(
|
||||
[self._source], robot_id="rl-train",
|
||||
spec=self._referee_spec(
|
||||
self.config, max_real_duration_s=self.real_budget_s),
|
||||
@@ -131,7 +130,7 @@ class LocalizationEnv(gym.Env):
|
||||
stress_mode=True)
|
||||
self._client = SimulatorClient(
|
||||
"rl-train", config=self.config,
|
||||
transport=self._LocalSimTransport(self._simulator),
|
||||
transport=self._local_sim_transport_cls(self._simulator),
|
||||
log=SessionLog())
|
||||
self._client.enter()
|
||||
outcome = self._client.measure(probe, self._source.channel)
|
||||
@@ -149,7 +148,9 @@ class LocalizationEnv(gym.Env):
|
||||
if self._scene_rng.random() < self.handoff_probability:
|
||||
angle = self._scene_rng.uniform(0, 2*math.pi)
|
||||
reach = self.config.field.omega_radius_m * math.sqrt(self._scene_rng.random())
|
||||
other = self.config.field.channel_min if self._channel != self.config.field.channel_min else self.config.field.channel_max
|
||||
other = (self.config.field.channel_min
|
||||
if self._channel != self.config.field.channel_min
|
||||
else self.config.field.channel_max)
|
||||
self._client.measure((reach*math.cos(angle), reach*math.sin(angle)), other)
|
||||
self._start_virtual_s = self._client.state.virtual_time_s
|
||||
return True
|
||||
@@ -325,5 +326,6 @@ class LocalizationEnv(gym.Env):
|
||||
return observation, float(reward), self._terminated, self._truncated, info
|
||||
|
||||
def close(self):
|
||||
if self._client is not None and self._client.state.entered and not self._client.state.exited:
|
||||
if (self._client is not None and self._client.state.entered
|
||||
and not self._client.state.exited):
|
||||
self._client.exit()
|
||||
|
||||
@@ -19,7 +19,7 @@ import math
|
||||
import numpy as np
|
||||
|
||||
from ..config import ProblemConfig, Vec2
|
||||
from ..planner import candidate_points, safe_disk_radius
|
||||
from ..planner import safe_disk_radius
|
||||
from ._constants import GLOBAL_DIM, LADDER_SLOTS, NUM_ACTIONS, OBS_DIM, PER_ACTION_DIM
|
||||
|
||||
__all__ = [
|
||||
@@ -100,7 +100,8 @@ def action_mask(region, position: Vec2, config: ProblemConfig, problem: int,
|
||||
Q3有距离保证盘时屏蔽盘外探点;Q4允许盘外探点,但绝不把距离保证
|
||||
当作朝向保证。训练与推理共用此函数,不读取源位置或朝向。
|
||||
"""
|
||||
key = lambda p: (round(p[0]*2), round(p[1]*2))
|
||||
def key(p):
|
||||
return (round(p[0]*2), round(p[1]*2))
|
||||
visited = {key(r['position']) for r in getattr(belief, 'measurements', [])}
|
||||
seen = set()
|
||||
center, _ = region.mec()
|
||||
|
||||
@@ -40,7 +40,8 @@ class PointSelectionPolicy:
|
||||
device = "cpu"
|
||||
self.device = device
|
||||
meta_path = path.with_suffix('.json')
|
||||
self.metadata = json.loads(meta_path.read_text(encoding='utf-8')) if meta_path.exists() else {}
|
||||
self.metadata = (json.loads(meta_path.read_text(encoding='utf-8'))
|
||||
if meta_path.exists() else {})
|
||||
self.feature_version = self.metadata.get('feature_version', 1)
|
||||
if self.feature_version not in (1, 2):
|
||||
raise RuntimeError('不支持的RL特征版本')
|
||||
@@ -78,7 +79,9 @@ class PointSelectionPolicy:
|
||||
region, position, channel, config, problem, blocked, belief,
|
||||
virtual_time_s=virtual_time_s,
|
||||
receiver_channel=receiver_channel if self.feature_version >= 2 else None)
|
||||
kwargs = {'action_masks': action_mask(region, position, config, problem, blocked, belief)} if self.masked else {}
|
||||
kwargs = ({'action_masks': action_mask(region, position, config, problem,
|
||||
blocked, belief)}
|
||||
if self.masked else {})
|
||||
action, _state = self.model.predict(observation, deterministic=True, **kwargs)
|
||||
action = int(action)
|
||||
candidates = candidate_actions(region, position, config, blocked)
|
||||
|
||||
@@ -0,0 +1,397 @@
|
||||
"""训练快照的纯函数派生:曲线统计、收敛判读、进程视图。
|
||||
|
||||
从 ``server.py`` 拆出来的两个理由:
|
||||
|
||||
1. 断开 ``demo`` ↔ ``server`` 的循环导入——离线预览(``demo.py``)只想复用
|
||||
这两个纯函数,没必要反向 import 整个 HTTP 服务器模块;
|
||||
2. 让"数值判读"可以脱离 HTTP 与子进程单独测试。
|
||||
|
||||
本模块只依赖标准库,不做任何 IO、不持有任何状态。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
__all__ = ["MAX_LOG_LINES", "MAX_METRICS", "finite_float",
|
||||
"summarize_convergence", "describe_process", "process_pid"]
|
||||
|
||||
MAX_LOG_LINES = 4000
|
||||
MAX_METRICS = 5000
|
||||
|
||||
_NUM_RE = re.compile(r"^[+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?$")
|
||||
|
||||
|
||||
def finite_float(value: Any) -> float | None:
|
||||
"""把任意输入转成有限浮点数;NaN / ±inf / 非数字一律返回 None。
|
||||
|
||||
旧实现用 ``value != value`` 判 NaN,pylint 报 ``comparison-with-itself``;
|
||||
这里用 ``math.isfinite`` 表达同一语义,意图更直白。
|
||||
"""
|
||||
if value is None or isinstance(value, bool):
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
value = float(value)
|
||||
if not math.isfinite(value):
|
||||
return None
|
||||
return value
|
||||
text = str(value).strip()
|
||||
if not _NUM_RE.match(text):
|
||||
return None
|
||||
try:
|
||||
return float(text)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _series(metrics: list[dict[str, Any]], key: str) -> list[tuple[float, float]]:
|
||||
points: list[tuple[float, float]] = []
|
||||
for point in metrics:
|
||||
value = finite_float(point.get(key))
|
||||
steps = finite_float(point.get("timesteps")) or finite_float(
|
||||
point.get("total_timesteps"))
|
||||
if value is None:
|
||||
continue
|
||||
points.append((steps if steps is not None else float(len(points) + 1), value))
|
||||
return points
|
||||
|
||||
|
||||
def _last_value(points: list[tuple[float, float]]) -> float | None:
|
||||
return points[-1][1] if points else None
|
||||
|
||||
|
||||
def _spread(points: list[tuple[float, float]], window: int = 10
|
||||
) -> tuple[float, float] | None:
|
||||
"""最近 window 个点的 (极差, 起点绝对值),用于判断"是否还在动"。"""
|
||||
sample = points[-window:]
|
||||
if len(sample) < 3:
|
||||
return None
|
||||
ys = [item[1] for item in sample]
|
||||
return max(ys) - min(ys), abs(ys[0])
|
||||
|
||||
|
||||
def _cv(points: list[tuple[float, float]], window: int = 10) -> float | None:
|
||||
"""最近 window 个点的变异系数(标准差/|均值|),越小越平稳。"""
|
||||
sample = [item[1] for item in points[-window:]]
|
||||
if len(sample) < 3:
|
||||
return None
|
||||
mean = sum(sample) / len(sample)
|
||||
if abs(mean) < 1e-12:
|
||||
return None
|
||||
var = sum((value - mean) ** 2 for value in sample) / len(sample)
|
||||
return (var ** 0.5) / abs(mean)
|
||||
|
||||
|
||||
def _slope(points: list[tuple[float, float]], window: int = 8) -> float | None:
|
||||
sample = points[-window:]
|
||||
if len(sample) < 3:
|
||||
return None
|
||||
xs = [item[0] for item in sample]
|
||||
ys = [item[1] for item in sample]
|
||||
n = len(xs)
|
||||
mean_x = sum(xs) / n
|
||||
mean_y = sum(ys) / n
|
||||
denom = sum((x - mean_x) ** 2 for x in xs)
|
||||
if abs(denom) < 1e-12:
|
||||
return None
|
||||
return sum((x - mean_x) * (y - mean_y) for x, y in zip(xs, ys)) / denom
|
||||
|
||||
|
||||
def _rate_per_1k(reward_points: list[tuple[float, float]],
|
||||
window: int = 8) -> float | None:
|
||||
"""最近 window 个点的奖励增益换算成"每 1000 环境步",便于读趋势。"""
|
||||
slope = _slope(reward_points, window)
|
||||
if slope is None:
|
||||
return None
|
||||
return slope * 1000.0
|
||||
|
||||
|
||||
def _fmt_num(value: float | None, digits: int = 4) -> str:
|
||||
if value is None:
|
||||
return "—"
|
||||
if value != 0 and abs(value) < 1e-3:
|
||||
return f"{value:.3e}"
|
||||
return f"{value:.{digits}g}"
|
||||
|
||||
|
||||
def _flatness(points: list[tuple[float, float]], window: int = 10
|
||||
) -> tuple[float, float] | None:
|
||||
"""窗口内"趋势位移 / 逐点噪声"比值:<1 表示看不出系统性变化。
|
||||
|
||||
判定收敛时只看斜率绝对值会把"很慢但持续的提升"误判为平台;这里把窗口
|
||||
对半切,用两半均值之差当趋势位移,用相邻点平均绝对差当噪声尺度。返回
|
||||
(比值, 噪声尺度),两者都小才是真的"又平又稳"。
|
||||
"""
|
||||
sample = points[-window:]
|
||||
if len(sample) < 4:
|
||||
return None
|
||||
ys = [item[1] for item in sample]
|
||||
half = len(ys) // 2
|
||||
shift = abs(sum(ys[half:]) / len(ys[half:]) - sum(ys[:half]) / len(ys[:half]))
|
||||
noise = sum(abs(b - a) for a, b in zip(ys, ys[1:])) / (len(ys) - 1)
|
||||
if noise <= 0:
|
||||
return (0.0, 0.0) if shift <= 0 else (float("inf"), 0.0)
|
||||
return shift / noise, noise
|
||||
|
||||
|
||||
def summarize_convergence(metrics: list[dict[str, Any]],
|
||||
monitor: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
"""给前端返回可解释的收敛状态与各参数最近值。
|
||||
|
||||
这是**启发式判读**,不替代正式学习曲线分析:只依据最近窗口的斜率、
|
||||
极差与变异系数给出"平坦/上升/未上升"的分类,并把同一组数字交给前端
|
||||
展示,便于人工复核。
|
||||
"""
|
||||
reward_points = _series(metrics, "ep_rew_mean")
|
||||
if not reward_points and monitor:
|
||||
reward_points = [(float(index), float(row.get("r", 0.0)))
|
||||
for index, row in enumerate(monitor)
|
||||
if finite_float(row.get("r")) is not None]
|
||||
kl_points = _series(metrics, "approx_kl")
|
||||
ev_points = _series(metrics, "explained_variance")
|
||||
vf_points = _series(metrics, "value_loss")
|
||||
pg_points = _series(metrics, "policy_gradient_loss")
|
||||
entropy_points = _series(metrics, "entropy_loss")
|
||||
fps_points = _series(metrics, "fps")
|
||||
|
||||
reward_slope = _slope(reward_points)
|
||||
reward_rate = _rate_per_1k(reward_points)
|
||||
spread = _spread(reward_points)
|
||||
cv = _cv(reward_points)
|
||||
latest_kl = _last_value(kl_points)
|
||||
latest_ev = _last_value(ev_points)
|
||||
latest_vf = _last_value(vf_points)
|
||||
latest_pg = _last_value(pg_points)
|
||||
latest_entropy = _last_value(entropy_points)
|
||||
latest_fps = _last_value(fps_points)
|
||||
latest_timesteps = metrics[-1].get("timesteps") if metrics else None
|
||||
if latest_timesteps is None and metrics:
|
||||
latest_timesteps = metrics[-1].get("total_timesteps")
|
||||
latest_reward = _last_value(reward_points)
|
||||
latest_len = _last_value(_series(metrics, "ep_len_mean"))
|
||||
latest_updates = _last_value(_series(metrics, "n_updates"))
|
||||
window = min(10, max(3, len(reward_points)))
|
||||
reward_window = [round(value, 6) for _, value in reward_points[-window:]]
|
||||
best_reward = max((value for _, value in reward_points), default=None)
|
||||
|
||||
notes: list[str] = []
|
||||
if not metrics and not monitor:
|
||||
status, tone = "等待首个 rollout", "gray"
|
||||
notes.append("训练进程启动后先输出环境初始化与首个 rollout 的进度行。")
|
||||
elif not metrics:
|
||||
status, tone = "已收到回合奖励,等待进度行", "warn"
|
||||
notes.append("Monitor CSV 已有回合记录,但尚未收到 --progress-json 进度行;"
|
||||
"首个 rollout 结束后会出现完整指标。")
|
||||
elif len(metrics) < 3:
|
||||
status, tone = "预热阶段", "warn"
|
||||
notes.append("已收到初始指标,需要更多 rollout 才能判断趋势。")
|
||||
else:
|
||||
tone, status = "ok", "收敛趋势良好"
|
||||
# 平台判据(三条同时成立):对半窗口看不出位移、窗口内波动小、
|
||||
# 逐点噪声本身也小。慢速但持续的提升会因第一条被排除。
|
||||
flat = _flatness(reward_points)
|
||||
plateau = False
|
||||
if flat is not None and cv is not None and latest_reward is not None:
|
||||
flat_ratio, flat_noise = flat
|
||||
scale = max(abs(latest_reward), 1e-9)
|
||||
plateau = (flat_ratio <= 1.0 and cv <= 0.15
|
||||
and flat_noise <= 0.02 * scale
|
||||
and (spread is None or spread[1] < 1e-12
|
||||
or spread[0] <= 0.15 * spread[1]))
|
||||
if latest_kl is not None and latest_kl > 0.05:
|
||||
status, tone = "KL 偏大,策略更新可能过猛", "bad"
|
||||
notes.append(f"approx_kl={latest_kl:.4f} > 0.05")
|
||||
elif latest_ev is not None and latest_ev < 0.0:
|
||||
status, tone = "价值函数尚未拟合", "warn"
|
||||
notes.append(f"explained_variance={latest_ev:.3f} < 0")
|
||||
elif reward_slope is not None and reward_slope < 0:
|
||||
status, tone = "奖励未形成上升趋势", "warn"
|
||||
notes.append(f"最近 {window} 点奖励斜率为负({_fmt_num(reward_slope)}/step)")
|
||||
elif plateau:
|
||||
status, tone = "已趋于平稳(疑似收敛)", "ok"
|
||||
notes.append("窗口内对半位移小于逐点噪声,且波动很小"
|
||||
f"(变异系数 {_fmt_num(cv, 3)})")
|
||||
else:
|
||||
notes.append(f"最近 {window} 点奖励斜率为正({_fmt_num(reward_slope)}/step)")
|
||||
if latest_kl is not None and latest_kl <= 0.02:
|
||||
notes.append(f"approx_kl={latest_kl:.4f} 较稳定")
|
||||
if latest_ev is not None:
|
||||
notes.append(f"explained_variance={latest_ev:.3f}")
|
||||
if cv is not None:
|
||||
notes.append(f"奖励变异系数={_fmt_num(cv, 3)}")
|
||||
if latest_entropy is not None and abs(latest_entropy) < 1e-3:
|
||||
notes.append("策略熵接近 0,探索可能过早停止")
|
||||
return {
|
||||
"status": status,
|
||||
"tone": tone,
|
||||
"notes": notes,
|
||||
"window": window,
|
||||
"latest_timesteps": latest_timesteps,
|
||||
"reward_slope": reward_slope,
|
||||
"reward_rate_per_1k": reward_rate,
|
||||
"reward_cv": cv,
|
||||
"reward_spread": spread[0] if spread else None,
|
||||
"reward_window": reward_window,
|
||||
"best_reward": best_reward,
|
||||
"latest_reward": latest_reward,
|
||||
"latest_ep_len": latest_len,
|
||||
"latest_updates": latest_updates,
|
||||
"latest_kl": latest_kl,
|
||||
"latest_explained_variance": latest_ev,
|
||||
"latest_value_loss": latest_vf,
|
||||
"latest_policy_loss": latest_pg,
|
||||
"latest_entropy_loss": latest_entropy,
|
||||
"latest_fps": latest_fps,
|
||||
"reward_points": len(reward_points),
|
||||
}
|
||||
|
||||
|
||||
# 快照状态 -> 界面展示用的进程语义(阶段名称、色调、是否终态)
|
||||
_STATUS_VIEW = {
|
||||
"created": ("已创建", "gray", False, "命令已构造,等待 spawn"),
|
||||
"running": ("运行中", "ok", False, "子进程持有控制台输出与独立进程组"),
|
||||
"paused": ("已暂停", "warn", False, "进程存活但停在 step 边界;点「继续」从暂停处恢复"),
|
||||
"stopping": ("停止中", "warn", False, "已请求优雅停止,等待子进程保存并退出"),
|
||||
"stopped": ("已手动停止", "warn", True, "收到停止请求后退出;脚本会在 step 边界保存模型与元数据"),
|
||||
"exited": ("正常结束", "ok", True, "退出码 0:训练与评估流程走完"),
|
||||
"failed": ("异常结束", "bad", True, "非零退出码或启动失败,见日志尾部"),
|
||||
}
|
||||
|
||||
_PHASES = [
|
||||
("created", "构造命令"),
|
||||
("spawn", "启动子进程"),
|
||||
("rollout", "采集 rollout"),
|
||||
("update", "PPO 更新"),
|
||||
("evaluate", "评估 / 保存"),
|
||||
("done", "收敛归档"),
|
||||
]
|
||||
|
||||
# 状态 -> 当前所处的阶段节点;节点之前为已完成,之后为未开始。
|
||||
_STATUS_PHASE = {
|
||||
"created": "created",
|
||||
"running": "update",
|
||||
"paused": "update",
|
||||
"stopping": "update",
|
||||
"stopped": "done",
|
||||
"exited": "done",
|
||||
"failed": "done",
|
||||
}
|
||||
|
||||
|
||||
def process_pid(process: subprocess.Popen | None) -> int | None:
|
||||
"""已退出进程的 pid 访问会抛 OSError;这里统一吞掉,只用于展示。"""
|
||||
if process is None:
|
||||
return None
|
||||
try:
|
||||
return int(process.pid)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def describe_process(snapshot: dict[str, Any]) -> dict[str, Any]:
|
||||
"""把 snapshot 派生为界面直接可渲染的进程对象。
|
||||
|
||||
包含:状态语义词、进度(步/百分比)、墙钟与剩余时间估计、最近吞吐、
|
||||
进程时间线节点。数字都取自 snapshot,前端不再重复累加,避免两个刷新
|
||||
频率不同的计数器互相打架。
|
||||
"""
|
||||
status = str(snapshot.get("status") or "created")
|
||||
label, tone, terminal, hint = _STATUS_VIEW.get(
|
||||
status, (status, "gray", False, ""))
|
||||
config = snapshot.get("config") or {}
|
||||
order = [key for key, _ in _PHASES]
|
||||
current = _STATUS_PHASE.get(status, "created")
|
||||
timeline = []
|
||||
for key, title in _PHASES:
|
||||
index = order.index(key)
|
||||
if index < order.index(current):
|
||||
state = "done"
|
||||
elif index == order.index(current):
|
||||
state = "active"
|
||||
else:
|
||||
state = "pending"
|
||||
timeline.append({"key": key, "label": title, "state": state})
|
||||
|
||||
metrics = snapshot.get("metrics") or []
|
||||
latest: dict[str, Any] = metrics[-1] if metrics else {}
|
||||
convergence = snapshot.get("convergence") or {}
|
||||
total_steps = finite_float(config.get("timesteps")) or 0.0
|
||||
# SB3 表格行可能没有 steps 字段,因此进度取"所有进度点的最大步数",
|
||||
# 而不是只看最后一个点——否则末行恰好缺字段时进度会回退到 0。
|
||||
done_steps = 0.0
|
||||
for point in metrics:
|
||||
value = (finite_float(point.get("timesteps"))
|
||||
or finite_float(point.get("total_timesteps")))
|
||||
if value is not None and value > done_steps:
|
||||
done_steps = value
|
||||
if done_steps == 0.0:
|
||||
done_steps = finite_float(convergence.get("latest_timesteps")) or 0.0
|
||||
# init/resume(课程迁移)时 num_timesteps 从上一阶段继续,进度必须按本次
|
||||
# 增量算;start_timesteps 由训练脚本的进度 JSON 提供,缺省按 0 处理。
|
||||
start_steps = 0.0
|
||||
for point in metrics:
|
||||
value = finite_float(point.get("start_timesteps"))
|
||||
if value is not None:
|
||||
start_steps = value
|
||||
break
|
||||
trained_steps = max(0.0, done_steps - start_steps)
|
||||
percent = (100.0 * trained_steps / total_steps) if total_steps > 0 else None
|
||||
if percent is not None:
|
||||
percent = max(0.0, min(100.0, percent))
|
||||
elapsed = float(snapshot.get("elapsed_s") or 0.0)
|
||||
rate = _latest_rate(metrics)
|
||||
remaining: float | None = None
|
||||
eta_ts: float | None = None
|
||||
if rate and rate > 0 and total_steps > 0 and trained_steps < total_steps:
|
||||
remaining = (total_steps - trained_steps) / rate
|
||||
eta_ts = float(snapshot.get("finished_at") or time.time()) + remaining
|
||||
if not terminal and remaining is None:
|
||||
cap = finite_float(config.get("max_wall_s")) or 0.0
|
||||
if cap > 0:
|
||||
remaining = max(0.0, cap - elapsed)
|
||||
return {
|
||||
"label": label,
|
||||
"tone": tone,
|
||||
"terminal": terminal,
|
||||
"hint": hint,
|
||||
"phase": current,
|
||||
"timeline": timeline,
|
||||
"percent": percent,
|
||||
"done_steps": done_steps,
|
||||
"start_steps": start_steps,
|
||||
"trained_steps": trained_steps,
|
||||
"total_steps": total_steps,
|
||||
"elapsed_s": round(elapsed, 3),
|
||||
"rate_per_s": rate,
|
||||
"remaining_s": remaining,
|
||||
"eta_epoch_s": eta_ts,
|
||||
"log_total": snapshot.get("log_total"),
|
||||
"log_capacity": MAX_LOG_LINES,
|
||||
"metric_total": snapshot.get("metric_total"),
|
||||
"latest": latest,
|
||||
}
|
||||
|
||||
|
||||
def _latest_rate(metrics: list[dict[str, Any]], window: int = 6) -> float | None:
|
||||
"""最近若干个**带 fps 的**进度点的平均吞吐(步/秒),用于剩余时间估计。
|
||||
|
||||
progress JSON 行不含 fps,只有 SB3 表格行有;若只看最后 window 个点,
|
||||
恰好落在若干 JSON 行之间时会得到 None,剩余时间就会退化成墙钟上限的
|
||||
粗糙估计。这里向前扫描直到拿到至少 1 个 fps(最多看 3*window 个点)。
|
||||
"""
|
||||
if not metrics:
|
||||
return None
|
||||
usable: list[float] = []
|
||||
for point in reversed(metrics[-window * 3:]):
|
||||
value = finite_float(point.get("fps"))
|
||||
if value and value > 0:
|
||||
usable.append(value)
|
||||
if len(usable) >= window:
|
||||
break
|
||||
if not usable:
|
||||
return None
|
||||
return sum(usable) / len(usable)
|
||||
@@ -15,6 +15,8 @@ import random
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from .analysis import describe_process, summarize_convergence
|
||||
|
||||
__all__ = ["DEMO_CONFIG", "DEMO_SCENARIOS", "demo_metrics", "demo_monitor",
|
||||
"demo_log_lines", "demo_snapshot"]
|
||||
|
||||
@@ -44,7 +46,7 @@ DEMO_STEPS = 48
|
||||
|
||||
|
||||
def _finite(value: float) -> float | None:
|
||||
if value != value or value in (float("inf"), float("-inf")):
|
||||
if not math.isfinite(value):
|
||||
return None
|
||||
return round(value, 8)
|
||||
|
||||
@@ -171,8 +173,6 @@ def demo_snapshot(*, log_tail: int = 400, scenario: str = "rising",
|
||||
(而不是刚好 100%);``scenario`` 选择收敛判读分支;``fps`` 决定剩余
|
||||
时间估计的吞吐量级。
|
||||
"""
|
||||
from .server import describe_process, summarize_convergence
|
||||
|
||||
now = time.time()
|
||||
ratio = min(1.0, max(0.05, float(done_ratio)))
|
||||
metrics = demo_metrics(count=max(6, int(round(DEMO_STEPS * ratio))),
|
||||
@@ -193,6 +193,7 @@ def demo_snapshot(*, log_tail: int = 400, scenario: str = "rising",
|
||||
"elapsed_s": 184.0,
|
||||
"returncode": None,
|
||||
"error": None,
|
||||
"forced_kill": False,
|
||||
"command": list(DEMO_COMMAND),
|
||||
"model": str(config["model"]),
|
||||
"config": config,
|
||||
@@ -203,6 +204,17 @@ def demo_snapshot(*, log_tail: int = 400, scenario: str = "rising",
|
||||
"metric_end": len(metrics),
|
||||
"metric_total": len(metrics),
|
||||
"monitor": monitor,
|
||||
"recovery": {
|
||||
"can_resume": True,
|
||||
"model": str(config["model"]),
|
||||
"timesteps": int(DEMO_STEPS * ratio * 12),
|
||||
"reason": "recovery",
|
||||
"updated_unix_s": now - 24.0,
|
||||
"source": "resume.json",
|
||||
"hint_file": str(config["model"]).rsplit("/", 1)[0] + "/resume.json",
|
||||
"candidates": 3,
|
||||
"preview": True,
|
||||
},
|
||||
"preview": True,
|
||||
"preview_scenario": scenario,
|
||||
}
|
||||
|
||||
@@ -163,6 +163,24 @@ CONFIG_FIELDS: list[dict[str, Any]] = [
|
||||
"label": "覆盖已有模型 --force", "default": False,
|
||||
"help": "输出路径已存在时必须勾选,否则训练脚本会拒绝覆盖。与续训互斥。",
|
||||
},
|
||||
# ---------------------------------------------------------- 恢复与落盘
|
||||
{
|
||||
"key": "recovery_every", "kind": "integer", "group": "recovery",
|
||||
"label": "滚动恢复点间隔 --recovery-every", "default": 2000,
|
||||
"help": "每 N 个环境步原子覆盖写 <checkpoints>/<stem>_latest.zip,"
|
||||
"训练开始前先写起步基线;急停(强杀)后最多丢 N 步。0=关闭。",
|
||||
},
|
||||
{
|
||||
"key": "checkpoint_every", "kind": "integer", "group": "recovery",
|
||||
"label": "历史 checkpoint 间隔 --checkpoint-every", "default": 20000,
|
||||
"help": "每 N 个环境步另留一份 <stem>_<步数>.zip 供返祖对比;"
|
||||
"0=只保留滚动恢复点。",
|
||||
},
|
||||
{
|
||||
"key": "checkpoint_keep", "kind": "integer", "group": "recovery",
|
||||
"label": "历史 checkpoint 保留数 --checkpoint-keep", "default": 3,
|
||||
"help": "只保留最近 K 个历史 checkpoint;滚动恢复点与起步基线不受影响。",
|
||||
},
|
||||
]
|
||||
|
||||
CONFIG_GROUPS: list[dict[str, str]] = [
|
||||
@@ -170,6 +188,7 @@ CONFIG_GROUPS: list[dict[str, str]] = [
|
||||
{"key": "ppo", "label": "PPO 超参数", "hint": "采样宽度与更新强度"},
|
||||
{"key": "runtime", "label": "运行环境", "hint": "并行、设备与低层执行器"},
|
||||
{"key": "scale", "label": "训练规模", "hint": "步数、墙钟上限与评估"},
|
||||
{"key": "recovery", "label": "恢复与落盘", "hint": "急停恢复点与历史 checkpoint"},
|
||||
{"key": "paths", "label": "模型与迁移", "hint": "产物路径、续训与课程迁移"},
|
||||
]
|
||||
|
||||
|
||||
+285
-401
@@ -4,7 +4,8 @@
|
||||
设计边界:
|
||||
- 只监听 127.0.0.1,只允许白名单静态文件;
|
||||
- 只允许一个训练任务,命令由服务端用参数列表构造,不经过 shell;
|
||||
- 支持启动/停止/状态查询;停止时发送 SIGTERM 到独立进程组;
|
||||
- 支持启动/暂停/继续/停止/状态查询;暂停与停止通过**控制文件**下发,
|
||||
POSIX 与 Windows 行为一致(详见 :mod:`mcm26b.process_control`);
|
||||
- 训练进程通过 ``--progress-json`` 输出单行 JSON 进度,服务端同时兼容
|
||||
Stable-Baselines3 的表格输出和 Monitor CSV。
|
||||
|
||||
@@ -12,15 +13,17 @@
|
||||
- ``GET /api/train/schema`` 字段/分组/预设/指标字典(``schema.py`` 单一出处)
|
||||
- ``GET /api/train/state`` 快照;``after_metric=N`` 只回增量指标
|
||||
- ``POST /api/train/start`` 以部分字段启动(其余取默认值)
|
||||
- ``POST /api/train/stop`` SIGTERM 独立进程组
|
||||
- ``POST /api/train/pause`` 请求训练进程在 step 边界挂起
|
||||
- ``POST /api/train/resume`` 解除挂起
|
||||
- ``POST /api/train/stop`` 请求优雅停止,超时后强杀进程树
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
@@ -32,15 +35,24 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from ..process_control import (DEFAULT_GRACE_S, spawn_kwargs,
|
||||
terminate_process_tree, write_control,
|
||||
write_json_atomic)
|
||||
from .analysis import (MAX_LOG_LINES, MAX_METRICS, describe_process, finite_float,
|
||||
process_pid, summarize_convergence)
|
||||
from .demo import DEMO_SCENARIOS, demo_snapshot
|
||||
from .schema import config_schema
|
||||
|
||||
__all__ = ["TrainingJob", "TrainingUI", "run_training_ui", "build_command",
|
||||
"summarize_convergence", "describe_process"]
|
||||
"summarize_convergence", "describe_process", "discover_recovery"]
|
||||
|
||||
STATIC_DIR = Path(__file__).resolve().parent / "static"
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
TRAIN_SCRIPT = REPO_ROOT / "scripts" / "train_hrl_high.py"
|
||||
# 控制文件集中放在 output/ 下(已 gitignore),job 之间互不干扰
|
||||
CONTROL_DIR = REPO_ROOT / "output" / "hrl" / "ui" / "control"
|
||||
# 从发出停止请求到强杀之间留给训练脚本收尾的秒数
|
||||
STOP_GRACE_S = DEFAULT_GRACE_S
|
||||
STATIC_FILES = {
|
||||
"/": ("index.html", "text/html; charset=utf-8"),
|
||||
"/index.html": ("index.html", "text/html; charset=utf-8"),
|
||||
@@ -49,10 +61,6 @@ STATIC_FILES = {
|
||||
"/style.css": ("style.css", "text/css; charset=utf-8"),
|
||||
}
|
||||
|
||||
MAX_LOG_LINES = 4000
|
||||
MAX_METRICS = 5000
|
||||
|
||||
_NUM_RE = re.compile(r"^[+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?$")
|
||||
_SB3_SECTIONS = ("rollout/", "time/", "train/", "system/")
|
||||
# 路径/字符串参数只拒绝控制字符(含 \r\n\t 与 DEL),不做 shell 元字符过滤:
|
||||
# 命令走参数列表、不经过 shell,这里只是防日志与命令行被注入控制序列。
|
||||
@@ -61,23 +69,6 @@ _SB3_SECTIONS = ("rollout/", "time/", "train/", "system/")
|
||||
_SAFE_ARG_RE = re.compile(r"^[^\x00-\x1f\x7f]{1,512}$")
|
||||
|
||||
|
||||
def _finite_float(value: Any) -> float | None:
|
||||
if value is None or isinstance(value, bool):
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
value = float(value)
|
||||
if value != value or value in (float("inf"), float("-inf")):
|
||||
return None
|
||||
return value
|
||||
text = str(value).strip()
|
||||
if not _NUM_RE.match(text):
|
||||
return None
|
||||
try:
|
||||
return float(text)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _normalize_metric(key: str) -> str:
|
||||
"""把 SB3 的 ``rollout/ep_rew_mean`` 与进度 JSON 的 ``ep_rew_mean``
|
||||
归一化到同一短名,便于前端绘制。"""
|
||||
@@ -109,6 +100,10 @@ class TrainConfig:
|
||||
max_wall_s: float = 1200.0
|
||||
local_selector: str = "geometry"
|
||||
local_model: str | None = None
|
||||
# 落盘/恢复:默认值与 scripts/train_hrl_high.py 保持一致
|
||||
checkpoint_every: int = 20000
|
||||
checkpoint_keep: int = 3
|
||||
recovery_every: int = 2000
|
||||
verbose: int = 1
|
||||
|
||||
@classmethod
|
||||
@@ -117,9 +112,11 @@ class TrainConfig:
|
||||
raise ValueError("请求体必须是 JSON 对象")
|
||||
defaults = cls()
|
||||
data: dict[str, Any] = {}
|
||||
for field in cls.__dataclass_fields__:
|
||||
if field in payload and payload[field] is not None:
|
||||
data[field] = payload[field]
|
||||
# 用 dataclasses.fields 而不是 cls.__dataclass_fields__:后者是内部
|
||||
# 实现细节,pylint 看不到它,也无法保证未来版本仍存在。
|
||||
for field in dataclasses.fields(cls):
|
||||
if field.name in payload and payload[field.name] is not None:
|
||||
data[field.name] = payload[field.name]
|
||||
cfg = cls(**{**defaults.__dict__, **data})
|
||||
cfg.validate()
|
||||
return cfg
|
||||
@@ -153,6 +150,12 @@ class TrainConfig:
|
||||
raise ValueError("max_wall_s 必须在 0..7200 秒")
|
||||
if int(self.eval_episodes) < 0:
|
||||
raise ValueError("eval_episodes 不能为负")
|
||||
if not 0 <= int(self.checkpoint_every) <= 10_000_000:
|
||||
raise ValueError("checkpoint_every 必须在 0..10,000,000")
|
||||
if not 1 <= int(self.checkpoint_keep) <= 100:
|
||||
raise ValueError("checkpoint_keep 必须在 1..100")
|
||||
if not 0 <= int(self.recovery_every) <= 10_000_000:
|
||||
raise ValueError("recovery_every 必须在 0..10,000,000")
|
||||
for label, value in (("model", self.model), ("init", self.init),
|
||||
("local_model", self.local_model)):
|
||||
if value is None:
|
||||
@@ -207,6 +210,9 @@ def build_command(config: TrainConfig) -> list[str]:
|
||||
"--n-epochs", str(config.n_epochs),
|
||||
"--max-wall-s", str(config.max_wall_s),
|
||||
"--local-selector", config.local_selector,
|
||||
"--checkpoint-every", str(config.checkpoint_every),
|
||||
"--checkpoint-keep", str(config.checkpoint_keep),
|
||||
"--recovery-every", str(config.recovery_every),
|
||||
"--progress-json",
|
||||
"--verbose", str(config.verbose),
|
||||
]
|
||||
@@ -223,25 +229,118 @@ def build_command(config: TrainConfig) -> list[str]:
|
||||
return cmd
|
||||
|
||||
|
||||
class TrainingJob:
|
||||
"""单个训练子进程的生命周期、日志和指标缓存。"""
|
||||
def discover_recovery(run_dir: Path, model_path: Path) -> dict[str, Any]:
|
||||
"""找出"现在就能继续训练"的那份产物。
|
||||
|
||||
def __init__(self, config: TrainConfig):
|
||||
优先级:
|
||||
1. ``<run_dir>/resume.json``——训练脚本写的权威指针(校验目标文件真实存在);
|
||||
2. 否则按 mtime 取最新:最终模型 / 滚动恢复点 / 历史 checkpoint / 异常兜底。
|
||||
|
||||
只认**已完整落盘**的文件:模型的 zip 与 json 都是原子替换写的,所以不会
|
||||
出现"文件在、但打不开"的半截快照。
|
||||
"""
|
||||
run_dir = Path(run_dir)
|
||||
model_path = Path(model_path)
|
||||
hint_file = run_dir / "resume.json"
|
||||
hint = _read_json_file(hint_file)
|
||||
|
||||
candidates: list[tuple[float, str, str]] = []
|
||||
|
||||
def _consider(path: Path, reason: str) -> None:
|
||||
try:
|
||||
stat = path.stat()
|
||||
except OSError:
|
||||
return
|
||||
if stat.st_size > 0:
|
||||
candidates.append((stat.st_mtime, str(path), reason))
|
||||
|
||||
_consider(model_path, "final_model")
|
||||
ckpt_dir = run_dir / "checkpoints"
|
||||
if ckpt_dir.is_dir():
|
||||
_consider(ckpt_dir / f"{model_path.stem}_ckpt_latest.zip", "recovery")
|
||||
for path in ckpt_dir.glob(f"{model_path.stem}_ckpt_[0-9]*.zip"):
|
||||
if path.stem.rsplit("_", 1)[-1].isdigit():
|
||||
_consider(path, "checkpoint")
|
||||
_consider(run_dir / f"{model_path.stem}_emergency.zip", "emergency")
|
||||
|
||||
state: dict[str, Any] = {
|
||||
"can_resume": False, "model": None, "timesteps": None,
|
||||
"reason": "none", "updated_unix_s": None, "source": "none",
|
||||
"hint_file": str(hint_file), "candidates": len(candidates),
|
||||
}
|
||||
if hint and hint.get("model") and Path(str(hint["model"])).is_file():
|
||||
state.update({
|
||||
"model": str(hint["model"]),
|
||||
"timesteps": _int_or_none(hint.get("timesteps")),
|
||||
"reason": str(hint.get("reason") or "resume_hint"),
|
||||
"updated_unix_s": _float_or_none(hint.get("created_unix_s")),
|
||||
"source": "resume.json",
|
||||
})
|
||||
elif candidates:
|
||||
mtime, path, reason = max(candidates, key=lambda item: item[0])
|
||||
state.update({
|
||||
"model": path, "reason": reason,
|
||||
"updated_unix_s": mtime, "source": "scan",
|
||||
})
|
||||
if state["model"]:
|
||||
state["can_resume"] = Path(str(state["model"])).is_file()
|
||||
return state
|
||||
|
||||
|
||||
def _read_json_file(path: Path) -> dict[str, Any] | None:
|
||||
"""读 JSON 文件;缺失或损坏返回 None(恢复指针不是关键路径,不能因此崩)。"""
|
||||
try:
|
||||
payload = json.loads(Path(path).read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError, UnicodeDecodeError):
|
||||
return None
|
||||
return payload if isinstance(payload, dict) else None
|
||||
|
||||
|
||||
def _int_or_none(value: Any) -> int | None:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _float_or_none(value: Any) -> float | None:
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
class TrainingJob:
|
||||
"""单个训练子进程的生命周期、日志和指标缓存。
|
||||
|
||||
``control_dir`` 是控制文件的落地目录;默认与模型同目录,服务端则统一放到
|
||||
``output/hrl/ui/control``(已被 gitignore)。
|
||||
"""
|
||||
|
||||
def __init__(self, config: TrainConfig, control_dir: Path | None = None):
|
||||
self.config = config
|
||||
self.job_id = f"job-{int(time.time() * 1000)}"
|
||||
self.command = config.command()
|
||||
self.control_path = (Path(control_dir) if control_dir is not None
|
||||
else config.model_path.parent) / \
|
||||
f"{self.job_id}.control.json"
|
||||
# 控制文件是暂停/停止的唯一可靠通道:Windows 上没有 SIGSTOP,
|
||||
# 且 os.kill(SIGTERM) 会退化成硬杀,信号无法承载"优雅收尾"语义。
|
||||
self.command = config.command() + ["--control-file", str(self.control_path)]
|
||||
self.created_at = time.time()
|
||||
self.started_at: float | None = None
|
||||
self.finished_at: float | None = None
|
||||
self.returncode: int | None = None
|
||||
self.status = "created" # created/running/exited/failed/stopped
|
||||
# created/running/paused/stopping/stopped/exited/failed
|
||||
self.status = "created"
|
||||
self.error: str | None = None
|
||||
self.forced_kill = False # 停止请求超时后是否动用了强杀
|
||||
self.process: subprocess.Popen | None = None
|
||||
self.logs: deque[str] = deque(maxlen=MAX_LOG_LINES)
|
||||
self.metrics: deque[dict[str, Any]] = deque(maxlen=MAX_METRICS)
|
||||
self.monitor: list[dict[str, Any]] = []
|
||||
self._lock = threading.RLock()
|
||||
self._reader: threading.Thread | None = None
|
||||
self._stopper: threading.Thread | None = None
|
||||
self._table_sections: list[tuple[str, dict[str, Any]]] = []
|
||||
self._table_steps: float | None = None # 当前表的 total_timesteps
|
||||
self._pending_table: dict[str, Any] | None = None # 未见刻度的半张表
|
||||
@@ -254,10 +353,13 @@ class TrainingJob:
|
||||
with self._lock:
|
||||
if self.process is not None and self.process.poll() is None:
|
||||
raise RuntimeError("已有训练任务在运行")
|
||||
# 先落一份"不暂停、不停止"的初始控制文件,训练脚本首步就能读到;
|
||||
# 否则脚本要等到第一次轮询才区分"文件缺失"与"无指令"。
|
||||
write_control(self.control_path, paused=False, stop=False)
|
||||
env = os.environ.copy()
|
||||
env["PYTHONUNBUFFERED"] = "1"
|
||||
env["HRL_UI_JOB_ID"] = self.job_id
|
||||
self.process = subprocess.Popen(
|
||||
self.process = subprocess.Popen( # pylint: disable=consider-using-with
|
||||
self.command,
|
||||
cwd=str(REPO_ROOT),
|
||||
stdout=subprocess.PIPE,
|
||||
@@ -265,7 +367,7 @@ class TrainingJob:
|
||||
text=True,
|
||||
bufsize=1,
|
||||
env=env,
|
||||
start_new_session=True,
|
||||
**spawn_kwargs(),
|
||||
)
|
||||
self.started_at = time.time()
|
||||
self.status = "running"
|
||||
@@ -273,18 +375,101 @@ class TrainingJob:
|
||||
target=self._read_loop, name=f"{self.job_id}-reader", daemon=True)
|
||||
self._reader.start()
|
||||
|
||||
def stop(self) -> bool:
|
||||
def pause(self) -> bool:
|
||||
"""请求训练进程在下一个 step 边界挂起(跨平台:控制文件)。"""
|
||||
with self._lock:
|
||||
process = self.process
|
||||
if process is None or process.poll() is not None:
|
||||
return False
|
||||
try:
|
||||
os.killpg(os.getpgid(process.pid), signal.SIGTERM)
|
||||
except (ProcessLookupError, PermissionError):
|
||||
process.terminate()
|
||||
self.status = "stopping"
|
||||
if self.status not in ("running", "paused"):
|
||||
return False
|
||||
write_control(self.control_path, paused=True, stop=False)
|
||||
self.status = "paused"
|
||||
return True
|
||||
|
||||
def resume(self) -> bool:
|
||||
"""解除挂起;训练从暂停的那个 step 继续。"""
|
||||
with self._lock:
|
||||
process = self.process
|
||||
if process is None or process.poll() is not None:
|
||||
return False
|
||||
if self.status not in ("paused", "running"):
|
||||
return False
|
||||
write_control(self.control_path, paused=False, stop=False)
|
||||
self.status = "running"
|
||||
return True
|
||||
|
||||
def stop(self) -> bool:
|
||||
"""请求优雅停止;超时后由看门狗强杀整棵进程树。
|
||||
|
||||
控制文件先于信号:Windows 上 SIGTERM 等价于 TerminateProcess,训练
|
||||
脚本"保存模型 + 写元数据"的收尾分支不会执行;控制文件则两个平台都能
|
||||
命中同一段协作式收尾。强杀放到独立线程,避免 HTTP 请求被 grace 等待卡住。
|
||||
"""
|
||||
with self._lock:
|
||||
process = self.process
|
||||
if process is None or process.poll() is not None:
|
||||
return False
|
||||
write_control(self.control_path, paused=False, stop=True)
|
||||
self.status = "stopping"
|
||||
if self._stopper is None or not self._stopper.is_alive():
|
||||
self._stopper = threading.Thread(
|
||||
target=self._stop_watchdog, args=(process,),
|
||||
name=f"{self.job_id}-stopper", daemon=True)
|
||||
self._stopper.start()
|
||||
return True
|
||||
|
||||
def _stop_watchdog(self, process: subprocess.Popen) -> None:
|
||||
"""等 grace 让协作式停止生效,超时强杀并记下恢复点。"""
|
||||
forced = terminate_process_tree(process, grace_s=STOP_GRACE_S)
|
||||
with self._lock:
|
||||
self.forced_kill = forced
|
||||
if forced and not self.error:
|
||||
self.error = (f"训练进程未在 {STOP_GRACE_S:.0f}s 内响应停止请求,"
|
||||
f"已强制结束进程树")
|
||||
if forced:
|
||||
self._write_forced_stop(process.pid)
|
||||
|
||||
def _write_forced_stop(self, pid: int) -> None:
|
||||
"""强杀后落一份 `<模型目录>/forced_stop.json`,写明"从哪继续"。
|
||||
|
||||
强杀没有给训练脚本任何收尾机会,所以这里由前端补上审计与恢复指引:
|
||||
磁盘上仍有原子落盘的滚动恢复点 / 历史 checkpoint,``recovery.model``
|
||||
就是可直接 ``--resume`` 的目标。
|
||||
"""
|
||||
run_dir = self.config.model_path.parent
|
||||
try:
|
||||
recovery = discover_recovery(run_dir, self.config.model_path)
|
||||
write_json_atomic(run_dir / "forced_stop.json", {
|
||||
"event": "forced_stop",
|
||||
"job_id": self.job_id,
|
||||
"pid": int(pid),
|
||||
"created_unix_s": time.time(),
|
||||
"reason": "grace_timeout",
|
||||
"grace_s": STOP_GRACE_S,
|
||||
"recovery": recovery,
|
||||
"note": ("训练进程未响应控制文件,已强杀整棵进程树;"
|
||||
"把 recovery.model 作为 --resume 目标即可继续"
|
||||
"(timesteps 为追加步数)"),
|
||||
})
|
||||
except OSError as exc: # 标记文件写不了也不能影响停止流程
|
||||
with self._lock:
|
||||
if not self.error:
|
||||
self.error = f"强杀后写恢复标记失败:{exc}"
|
||||
|
||||
def wait_for_exit(self, timeout: float | None = None) -> bool:
|
||||
"""等待停止流程收尾(优雅退出或强杀);供前端进程退出前调用。
|
||||
|
||||
训练子进程在独立进程组里,前端先退出并不会带走它。若不在这里等,
|
||||
前端一退出,子进程会成为孤儿——虽然控制文件仍会让它自行收尾,但
|
||||
日志与退出码就没人接了。
|
||||
"""
|
||||
thread = self._stopper
|
||||
if thread is not None:
|
||||
thread.join(timeout)
|
||||
process = self.process
|
||||
return process is None or process.poll() is not None
|
||||
|
||||
def _read_loop(self) -> None:
|
||||
process = self.process
|
||||
assert process is not None and process.stdout is not None
|
||||
@@ -315,10 +500,38 @@ class TrainingJob:
|
||||
def _on_line(self, line: str) -> None:
|
||||
with self._lock:
|
||||
self.logs.append(line)
|
||||
self._sync_control_events(line)
|
||||
self._parse_progress_json(line)
|
||||
self._parse_sb3_table(line)
|
||||
self._refresh_monitor()
|
||||
|
||||
def _sync_control_events(self, line: str) -> None:
|
||||
"""用训练脚本的回显校正前端状态。
|
||||
|
||||
正常情况下状态由前端自己设置;但外部直接改控制文件、或脚本比前端更早
|
||||
感知到指令时,以脚本回显的 ``paused`` / ``resumed`` / ``stop_requested``
|
||||
为准更真实。
|
||||
"""
|
||||
stripped = line.strip()
|
||||
if not stripped.startswith("{") or '"event"' not in stripped:
|
||||
return
|
||||
try:
|
||||
payload = json.loads(stripped)
|
||||
except ValueError:
|
||||
return
|
||||
if not isinstance(payload, dict):
|
||||
return
|
||||
event = payload.get("event")
|
||||
if event not in ("paused", "resumed", "stop_requested"):
|
||||
return
|
||||
with self._lock:
|
||||
if event == "paused" and self.status == "running":
|
||||
self.status = "paused"
|
||||
elif event == "resumed" and self.status == "paused":
|
||||
self.status = "running"
|
||||
elif event == "stop_requested" and self.status in ("running", "paused"):
|
||||
self.status = "stopping"
|
||||
|
||||
# ------------------------------------------------------------ parsers
|
||||
def _parse_progress_json(self, line: str) -> None:
|
||||
stripped = line.strip()
|
||||
@@ -332,9 +545,9 @@ class TrainingJob:
|
||||
return
|
||||
point = {_normalize_metric(str(key)): value
|
||||
for key, value in payload.items() if key != "event"}
|
||||
steps = _finite_float(point.get("timesteps"))
|
||||
steps = finite_float(point.get("timesteps"))
|
||||
if steps is None:
|
||||
steps = _finite_float(point.get("total_timesteps"))
|
||||
steps = finite_float(point.get("total_timesteps"))
|
||||
if steps is not None:
|
||||
with self._lock:
|
||||
self._last_json_steps = steps
|
||||
@@ -371,7 +584,7 @@ class TrainingJob:
|
||||
self._table_sections.append((raw_key, {}))
|
||||
return
|
||||
key = _normalize_metric(raw_key)
|
||||
value = _finite_float(raw_value)
|
||||
value = finite_float(raw_value)
|
||||
if not key or value is None:
|
||||
return
|
||||
with self._lock:
|
||||
@@ -401,7 +614,7 @@ class TrainingJob:
|
||||
for _name, items in sections:
|
||||
table.update(items)
|
||||
if steps is None:
|
||||
steps = _finite_float(table.get("total_timesteps"))
|
||||
steps = finite_float(table.get("total_timesteps"))
|
||||
if steps is None:
|
||||
# 半张表:交给调用方决定是否已确认结束(终止时按末尾兜底提交)
|
||||
with self._lock:
|
||||
@@ -432,7 +645,7 @@ class TrainingJob:
|
||||
def _append_metric(self, point: dict[str, Any]) -> None:
|
||||
clean: dict[str, Any] = {}
|
||||
for key, value in point.items():
|
||||
number = _finite_float(value)
|
||||
number = finite_float(value)
|
||||
clean[key] = number if number is not None else value
|
||||
# 补齐 timesteps:progress JSON 优先,SB3 表用 total_timesteps。
|
||||
if "timesteps" not in clean and "total_timesteps" in clean:
|
||||
@@ -471,7 +684,7 @@ class TrainingJob:
|
||||
row = {}
|
||||
valid = True
|
||||
for key, raw in zip(header, values):
|
||||
number = _finite_float(raw)
|
||||
number = finite_float(raw)
|
||||
if number is None:
|
||||
valid = False
|
||||
break
|
||||
@@ -505,17 +718,17 @@ class TrainingJob:
|
||||
config = dict(self.config.__dict__)
|
||||
command = list(self.command)
|
||||
model_path = str(self.config.model_path)
|
||||
timesteps = int(self.config.timesteps)
|
||||
payload = {
|
||||
"job_id": self.job_id,
|
||||
"status": self.status,
|
||||
"pid": _pid(self.process),
|
||||
"pid": process_pid(self.process),
|
||||
"created_at": self.created_at,
|
||||
"started_at": self.started_at,
|
||||
"finished_at": self.finished_at,
|
||||
"elapsed_s": round(elapsed, 3),
|
||||
"returncode": self.returncode,
|
||||
"error": self.error,
|
||||
"forced_kill": self.forced_kill,
|
||||
"command": command,
|
||||
"model": model_path,
|
||||
"config": config,
|
||||
@@ -529,356 +742,12 @@ class TrainingJob:
|
||||
}
|
||||
payload["convergence"] = summarize_convergence(metrics, monitor)
|
||||
payload["process"] = describe_process(payload)
|
||||
# 恢复指针在锁外做文件扫描:即使训练被强杀,这里也能给出可续训的产物
|
||||
payload["recovery"] = discover_recovery(
|
||||
self.config.model_path.parent, self.config.model_path)
|
||||
return payload
|
||||
|
||||
|
||||
def _series(metrics: list[dict[str, Any]], key: str) -> list[tuple[float, float]]:
|
||||
points: list[tuple[float, float]] = []
|
||||
for point in metrics:
|
||||
value = _finite_float(point.get(key))
|
||||
steps = _finite_float(point.get("timesteps")) or _finite_float(
|
||||
point.get("total_timesteps"))
|
||||
if value is None:
|
||||
continue
|
||||
points.append((steps if steps is not None else float(len(points) + 1), value))
|
||||
return points
|
||||
|
||||
|
||||
def _last_value(points: list[tuple[float, float]]) -> float | None:
|
||||
return points[-1][1] if points else None
|
||||
|
||||
|
||||
def _spread(points: list[tuple[float, float]], window: int = 10
|
||||
) -> tuple[float, float] | None:
|
||||
"""最近 window 个点的 (极差, 起点绝对值),用于判断"是否还在动"。"""
|
||||
sample = points[-window:]
|
||||
if len(sample) < 3:
|
||||
return None
|
||||
ys = [item[1] for item in sample]
|
||||
return max(ys) - min(ys), abs(ys[0])
|
||||
|
||||
|
||||
def _cv(points: list[tuple[float, float]], window: int = 10) -> float | None:
|
||||
"""最近 window 个点的变异系数(标准差/|均值|),越小越平稳。"""
|
||||
sample = [item[1] for item in points[-window:]]
|
||||
if len(sample) < 3:
|
||||
return None
|
||||
mean = sum(sample) / len(sample)
|
||||
if abs(mean) < 1e-12:
|
||||
return None
|
||||
var = sum((value - mean) ** 2 for value in sample) / len(sample)
|
||||
return (var ** 0.5) / abs(mean)
|
||||
|
||||
|
||||
def _slope(points: list[tuple[float, float]], window: int = 8) -> float | None:
|
||||
sample = points[-window:]
|
||||
if len(sample) < 3:
|
||||
return None
|
||||
xs = [item[0] for item in sample]
|
||||
ys = [item[1] for item in sample]
|
||||
n = len(xs)
|
||||
mean_x = sum(xs) / n
|
||||
mean_y = sum(ys) / n
|
||||
denom = sum((x - mean_x) ** 2 for x in xs)
|
||||
if abs(denom) < 1e-12:
|
||||
return None
|
||||
return sum((x - mean_x) * (y - mean_y) for x, y in zip(xs, ys)) / denom
|
||||
|
||||
|
||||
def _rate_per_1k(reward_points: list[tuple[float, float]],
|
||||
window: int = 8) -> float | None:
|
||||
"""最近 window 个点的奖励增益换算成"每 1000 环境步",便于读趋势。"""
|
||||
slope = _slope(reward_points, window)
|
||||
if slope is None:
|
||||
return None
|
||||
return slope * 1000.0
|
||||
|
||||
|
||||
def _fmt_num(value: float | None, digits: int = 4) -> str:
|
||||
if value is None:
|
||||
return "—"
|
||||
if value != 0 and abs(value) < 1e-3:
|
||||
return f"{value:.3e}"
|
||||
return f"{value:.{digits}g}"
|
||||
|
||||
|
||||
def _flatness(points: list[tuple[float, float]], window: int = 10
|
||||
) -> tuple[float, float] | None:
|
||||
"""窗口内"趋势位移 / 逐点噪声"比值:<1 表示看不出系统性变化。
|
||||
|
||||
判定收敛时只看斜率绝对值会把"很慢但持续的提升"误判为平台;这里把窗口
|
||||
对半切,用两半均值之差当趋势位移,用相邻点平均绝对差当噪声尺度。返回
|
||||
(比值, 噪声尺度),两者都小才是真的"又平又稳"。
|
||||
"""
|
||||
sample = points[-window:]
|
||||
if len(sample) < 4:
|
||||
return None
|
||||
ys = [item[1] for item in sample]
|
||||
half = len(ys) // 2
|
||||
shift = abs(sum(ys[half:]) / len(ys[half:]) - sum(ys[:half]) / len(ys[:half]))
|
||||
noise = sum(abs(b - a) for a, b in zip(ys, ys[1:])) / (len(ys) - 1)
|
||||
if noise <= 0:
|
||||
return (0.0, 0.0) if shift <= 0 else (float("inf"), 0.0)
|
||||
return shift / noise, noise
|
||||
|
||||
|
||||
def summarize_convergence(metrics: list[dict[str, Any]],
|
||||
monitor: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
"""给前端返回可解释的收敛状态与各参数最近值。
|
||||
|
||||
这是**启发式判读**,不替代正式学习曲线分析:只依据最近窗口的斜率、
|
||||
极差与变异系数给出"平坦/上升/未上升"的分类,并把同一组数字交给前端
|
||||
展示,便于人工复核。
|
||||
"""
|
||||
reward_points = _series(metrics, "ep_rew_mean")
|
||||
if not reward_points and monitor:
|
||||
reward_points = [(float(index), float(row.get("r", 0.0)))
|
||||
for index, row in enumerate(monitor)
|
||||
if _finite_float(row.get("r")) is not None]
|
||||
kl_points = _series(metrics, "approx_kl")
|
||||
ev_points = _series(metrics, "explained_variance")
|
||||
vf_points = _series(metrics, "value_loss")
|
||||
pg_points = _series(metrics, "policy_gradient_loss")
|
||||
entropy_points = _series(metrics, "entropy_loss")
|
||||
fps_points = _series(metrics, "fps")
|
||||
|
||||
reward_slope = _slope(reward_points)
|
||||
reward_rate = _rate_per_1k(reward_points)
|
||||
spread = _spread(reward_points)
|
||||
cv = _cv(reward_points)
|
||||
latest_kl = _last_value(kl_points)
|
||||
latest_ev = _last_value(ev_points)
|
||||
latest_vf = _last_value(vf_points)
|
||||
latest_pg = _last_value(pg_points)
|
||||
latest_entropy = _last_value(entropy_points)
|
||||
latest_fps = _last_value(fps_points)
|
||||
latest_timesteps = metrics[-1].get("timesteps") if metrics else None
|
||||
if latest_timesteps is None and metrics:
|
||||
latest_timesteps = metrics[-1].get("total_timesteps")
|
||||
latest_reward = _last_value(reward_points)
|
||||
latest_len = _last_value(_series(metrics, "ep_len_mean"))
|
||||
latest_updates = _last_value(_series(metrics, "n_updates"))
|
||||
window = min(10, max(3, len(reward_points)))
|
||||
reward_window = [round(value, 6) for _, value in reward_points[-window:]]
|
||||
best_reward = max((value for _, value in reward_points), default=None)
|
||||
|
||||
notes: list[str] = []
|
||||
if not metrics and not monitor:
|
||||
status, tone = "等待首个 rollout", "gray"
|
||||
notes.append("训练进程启动后先输出环境初始化与首个 rollout 的进度行。")
|
||||
elif not metrics:
|
||||
status, tone = "已收到回合奖励,等待进度行", "warn"
|
||||
notes.append("Monitor CSV 已有回合记录,但尚未收到 --progress-json 进度行;"
|
||||
"首个 rollout 结束后会出现完整指标。")
|
||||
elif len(metrics) < 3:
|
||||
status, tone = "预热阶段", "warn"
|
||||
notes.append("已收到初始指标,需要更多 rollout 才能判断趋势。")
|
||||
else:
|
||||
tone, status = "ok", "收敛趋势良好"
|
||||
# 平台判据(三条同时成立):对半窗口看不出位移、窗口内波动小、
|
||||
# 逐点噪声本身也小。慢速但持续的提升会因第一条被排除。
|
||||
flat = _flatness(reward_points)
|
||||
plateau = False
|
||||
if flat is not None and cv is not None and latest_reward is not None:
|
||||
flat_ratio, flat_noise = flat
|
||||
scale = max(abs(latest_reward), 1e-9)
|
||||
plateau = (flat_ratio <= 1.0 and cv <= 0.15
|
||||
and flat_noise <= 0.02 * scale
|
||||
and (spread is None or spread[1] < 1e-12
|
||||
or spread[0] <= 0.15 * spread[1]))
|
||||
if latest_kl is not None and latest_kl > 0.05:
|
||||
status, tone = "KL 偏大,策略更新可能过猛", "bad"
|
||||
notes.append(f"approx_kl={latest_kl:.4f} > 0.05")
|
||||
elif latest_ev is not None and latest_ev < 0.0:
|
||||
status, tone = "价值函数尚未拟合", "warn"
|
||||
notes.append(f"explained_variance={latest_ev:.3f} < 0")
|
||||
elif reward_slope is not None and reward_slope < 0:
|
||||
status, tone = "奖励未形成上升趋势", "warn"
|
||||
notes.append(f"最近 {window} 点奖励斜率为负({_fmt_num(reward_slope)}/step)")
|
||||
elif plateau:
|
||||
status, tone = "已趋于平稳(疑似收敛)", "ok"
|
||||
notes.append("窗口内对半位移小于逐点噪声,且波动很小"
|
||||
f"(变异系数 {_fmt_num(cv, 3)})")
|
||||
else:
|
||||
notes.append(f"最近 {window} 点奖励斜率为正({_fmt_num(reward_slope)}/step)")
|
||||
if latest_kl is not None and latest_kl <= 0.02:
|
||||
notes.append(f"approx_kl={latest_kl:.4f} 较稳定")
|
||||
if latest_ev is not None:
|
||||
notes.append(f"explained_variance={latest_ev:.3f}")
|
||||
if cv is not None:
|
||||
notes.append(f"奖励变异系数={_fmt_num(cv, 3)}")
|
||||
if latest_entropy is not None and abs(latest_entropy) < 1e-3:
|
||||
notes.append("策略熵接近 0,探索可能过早停止")
|
||||
return {
|
||||
"status": status,
|
||||
"tone": tone,
|
||||
"notes": notes,
|
||||
"window": window,
|
||||
"latest_timesteps": latest_timesteps,
|
||||
"reward_slope": reward_slope,
|
||||
"reward_rate_per_1k": reward_rate,
|
||||
"reward_cv": cv,
|
||||
"reward_spread": spread[0] if spread else None,
|
||||
"reward_window": reward_window,
|
||||
"best_reward": best_reward,
|
||||
"latest_reward": latest_reward,
|
||||
"latest_ep_len": latest_len,
|
||||
"latest_updates": latest_updates,
|
||||
"latest_kl": latest_kl,
|
||||
"latest_explained_variance": latest_ev,
|
||||
"latest_value_loss": latest_vf,
|
||||
"latest_policy_loss": latest_pg,
|
||||
"latest_entropy_loss": latest_entropy,
|
||||
"latest_fps": latest_fps,
|
||||
"reward_points": len(reward_points),
|
||||
}
|
||||
|
||||
|
||||
# 快照状态 -> 界面展示用的进程语义(阶段名称、色调、是否终态)
|
||||
_STATUS_VIEW = {
|
||||
"created": ("已创建", "gray", False, "命令已构造,等待 spawn"),
|
||||
"running": ("运行中", "ok", False, "子进程持有控制台输出与独立进程组"),
|
||||
"stopping": ("停止中", "warn", False, "已发送 SIGTERM,等待子进程退出"),
|
||||
"stopped": ("已手动停止", "warn", True, "收到停止信号后退出;脚本会在 step 边界保存模型与元数据"),
|
||||
"exited": ("正常结束", "ok", True, "退出码 0:训练与评估流程走完"),
|
||||
"failed": ("异常结束", "bad", True, "非零退出码或启动失败,见日志尾部"),
|
||||
}
|
||||
|
||||
_PHASES = [
|
||||
("created", "构造命令"),
|
||||
("spawn", "启动子进程"),
|
||||
("rollout", "采集 rollout"),
|
||||
("update", "PPO 更新"),
|
||||
("evaluate", "评估 / 保存"),
|
||||
("done", "收敛归档"),
|
||||
]
|
||||
|
||||
# 状态 -> 当前所处的阶段节点;节点之前为已完成,之后为未开始。
|
||||
_STATUS_PHASE = {
|
||||
"created": "created",
|
||||
"running": "update",
|
||||
"stopping": "update",
|
||||
"stopped": "done",
|
||||
"exited": "done",
|
||||
"failed": "done",
|
||||
}
|
||||
|
||||
|
||||
def _pid(process: subprocess.Popen | None) -> int | None:
|
||||
"""已退出进程的 pid 访问会抛 OSError;这里统一吞掉,只用于展示。"""
|
||||
if process is None:
|
||||
return None
|
||||
try:
|
||||
return int(process.pid)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def describe_process(snapshot: dict[str, Any]) -> dict[str, Any]:
|
||||
"""把 snapshot 派生为界面直接可渲染的进程对象。
|
||||
|
||||
包含:状态语义词、进度(步/百分比)、墙钟与剩余时间估计、最近吞吐、
|
||||
进程时间线节点。数字都取自 snapshot,前端不再重复累加,避免两个刷新
|
||||
频率不同的计数器互相打架。
|
||||
"""
|
||||
status = str(snapshot.get("status") or "created")
|
||||
label, tone, terminal, hint = _STATUS_VIEW.get(
|
||||
status, (status, "gray", False, ""))
|
||||
config = snapshot.get("config") or {}
|
||||
order = [key for key, _ in _PHASES]
|
||||
current = _STATUS_PHASE.get(status, "created")
|
||||
timeline = []
|
||||
for key, title in _PHASES:
|
||||
index = order.index(key)
|
||||
if index < order.index(current):
|
||||
state = "done"
|
||||
elif index == order.index(current):
|
||||
state = "active"
|
||||
else:
|
||||
state = "pending"
|
||||
timeline.append({"key": key, "label": title, "state": state})
|
||||
|
||||
metrics = snapshot.get("metrics") or []
|
||||
latest: dict[str, Any] = metrics[-1] if metrics else {}
|
||||
convergence = snapshot.get("convergence") or {}
|
||||
total_steps = _finite_float(config.get("timesteps")) or 0.0
|
||||
# SB3 表格行可能没有 steps 字段,因此进度取"所有进度点的最大步数",
|
||||
# 而不是只看最后一个点——否则末行恰好缺字段时进度会回退到 0。
|
||||
done_steps = 0.0
|
||||
for point in metrics:
|
||||
value = (_finite_float(point.get("timesteps"))
|
||||
or _finite_float(point.get("total_timesteps")))
|
||||
if value is not None and value > done_steps:
|
||||
done_steps = value
|
||||
if done_steps == 0.0:
|
||||
done_steps = _finite_float(convergence.get("latest_timesteps")) or 0.0
|
||||
# init/resume(课程迁移)时 num_timesteps 从上一阶段继续,进度必须按本次
|
||||
# 增量算;start_timesteps 由训练脚本的进度 JSON 提供,缺省按 0 处理。
|
||||
start_steps = 0.0
|
||||
for point in metrics:
|
||||
value = _finite_float(point.get("start_timesteps"))
|
||||
if value is not None:
|
||||
start_steps = value
|
||||
break
|
||||
trained_steps = max(0.0, done_steps - start_steps)
|
||||
percent = (100.0 * trained_steps / total_steps) if total_steps > 0 else None
|
||||
if percent is not None:
|
||||
percent = max(0.0, min(100.0, percent))
|
||||
elapsed = float(snapshot.get("elapsed_s") or 0.0)
|
||||
rate = _latest_rate(metrics)
|
||||
remaining: float | None = None
|
||||
eta_ts: float | None = None
|
||||
if rate and rate > 0 and total_steps > 0 and trained_steps < total_steps:
|
||||
remaining = (total_steps - trained_steps) / rate
|
||||
eta_ts = float(snapshot.get("finished_at") or time.time()) + remaining
|
||||
if not terminal and remaining is None:
|
||||
cap = _finite_float(config.get("max_wall_s")) or 0.0
|
||||
if cap > 0:
|
||||
remaining = max(0.0, cap - elapsed)
|
||||
return {
|
||||
"label": label,
|
||||
"tone": tone,
|
||||
"terminal": terminal,
|
||||
"hint": hint,
|
||||
"phase": current,
|
||||
"timeline": timeline,
|
||||
"percent": percent,
|
||||
"done_steps": done_steps,
|
||||
"start_steps": start_steps,
|
||||
"trained_steps": trained_steps,
|
||||
"total_steps": total_steps,
|
||||
"elapsed_s": round(elapsed, 3),
|
||||
"rate_per_s": rate,
|
||||
"remaining_s": remaining,
|
||||
"eta_epoch_s": eta_ts,
|
||||
"log_total": snapshot.get("log_total"),
|
||||
"log_capacity": MAX_LOG_LINES,
|
||||
"metric_total": snapshot.get("metric_total"),
|
||||
"latest": latest,
|
||||
}
|
||||
|
||||
|
||||
def _latest_rate(metrics: list[dict[str, Any]], window: int = 6) -> float | None:
|
||||
"""最近若干个**带 fps 的**进度点的平均吞吐(步/秒),用于剩余时间估计。
|
||||
|
||||
progress JSON 行不含 fps,只有 SB3 表格行有;若只看最后 window 个点,
|
||||
恰好落在若干 JSON 行之间时会得到 None,剩余时间就会退化成墙钟上限的
|
||||
粗糙估计。这里向前扫描直到拿到至少 1 个 fps(最多看 3*window 个点)。
|
||||
"""
|
||||
if not metrics:
|
||||
return None
|
||||
usable: list[float] = []
|
||||
for point in reversed(metrics[-window * 3:]):
|
||||
value = _finite_float(point.get("fps"))
|
||||
if value and value > 0:
|
||||
usable.append(value)
|
||||
if len(usable) >= window:
|
||||
break
|
||||
if not usable:
|
||||
return None
|
||||
return sum(usable) / len(usable)
|
||||
|
||||
|
||||
class TrainingUI:
|
||||
"""训练前端管理器:持有唯一 TrainingJob 与 HTTP 服务器。"""
|
||||
|
||||
@@ -901,9 +770,9 @@ class TrainingUI:
|
||||
config = TrainConfig.from_payload(payload)
|
||||
with self._lock:
|
||||
if self.job is not None and self.job.status in (
|
||||
"created", "running", "stopping"):
|
||||
"created", "running", "paused", "stopping"):
|
||||
raise RuntimeError("已有训练任务在运行,请先停止")
|
||||
self.job = TrainingJob(config)
|
||||
self.job = TrainingJob(config, control_dir=CONTROL_DIR)
|
||||
try:
|
||||
self.job.start()
|
||||
except Exception:
|
||||
@@ -913,6 +782,14 @@ class TrainingUI:
|
||||
# 与 snapshot() 同形:HTTP 层直接把返回值当响应体,不再二次拆包
|
||||
return {"job": self.job.snapshot(log_tail=1)}
|
||||
|
||||
def pause(self) -> bool:
|
||||
with self._lock:
|
||||
return bool(self.job and self.job.pause())
|
||||
|
||||
def resume(self) -> bool:
|
||||
with self._lock:
|
||||
return bool(self.job and self.job.resume())
|
||||
|
||||
def stop(self) -> bool:
|
||||
with self._lock:
|
||||
return bool(self.job and self.job.stop())
|
||||
@@ -938,7 +815,10 @@ class TrainingUI:
|
||||
self._serving = False
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self.stop()
|
||||
job = self.job
|
||||
if job is not None and job.stop():
|
||||
# 留出收尾落盘的时间,避免前端先退出把训练子进程留成孤儿
|
||||
job.wait_for_exit(timeout=STOP_GRACE_S + 5.0)
|
||||
if self._serving:
|
||||
self.httpd.shutdown()
|
||||
self.httpd.server_close()
|
||||
@@ -948,8 +828,8 @@ class _UIHandler(BaseHTTPRequestHandler):
|
||||
protocol_version = "HTTP/1.1"
|
||||
server_version = "MCM26B-Training-UI/1.0"
|
||||
|
||||
def log_message(self, format, *args): # noqa: A002
|
||||
pass
|
||||
def log_message(self, format, *args): # noqa: A002 # pylint: disable=redefined-builtin
|
||||
"""覆写 stdlib 签名以吞掉访问日志;参数名沿用 ``format`` 不能改。"""
|
||||
|
||||
@property
|
||||
def ui(self) -> TrainingUI:
|
||||
@@ -999,6 +879,10 @@ class _UIHandler(BaseHTTPRequestHandler):
|
||||
try:
|
||||
if parts.path == "/api/train/start":
|
||||
self._send_json(200, self.ui.start(payload))
|
||||
elif parts.path == "/api/train/pause":
|
||||
self._send_json(200, {"paused": self.ui.pause()})
|
||||
elif parts.path == "/api/train/resume":
|
||||
self._send_json(200, {"resumed": self.ui.resume()})
|
||||
elif parts.path == "/api/train/stop":
|
||||
self._send_json(200, {"stopped": self.ui.stop()})
|
||||
else:
|
||||
|
||||
@@ -23,6 +23,8 @@ const state = {
|
||||
chartKey: "ep_rew_mean",
|
||||
jobKey: null, // job_id 变化时清空本地缓存
|
||||
lastLogSignature: "",
|
||||
lastStatus: null, // 最近一次快照的状态:决定暂停按钮是"暂停"还是"继续"
|
||||
lastJob: null, // 最近一次快照:恢复按钮需要读 job.recovery
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------- 工具
|
||||
@@ -285,9 +287,11 @@ function absorbMetrics(job) {
|
||||
// ---------------------------------------------------------------- 渲染
|
||||
function render(job) {
|
||||
if (!job) {
|
||||
state.lastJob = null;
|
||||
renderIdle();
|
||||
return;
|
||||
}
|
||||
state.lastJob = job;
|
||||
if (job.job_id !== state.jobKey) {
|
||||
state.jobKey = job.job_id;
|
||||
state.metrics = [];
|
||||
@@ -302,6 +306,7 @@ function render(job) {
|
||||
renderLog(job);
|
||||
renderEpisodes(job);
|
||||
renderBanner(job);
|
||||
renderRecovery(job);
|
||||
renderJobLine(job);
|
||||
}
|
||||
|
||||
@@ -319,6 +324,10 @@ function renderIdle() {
|
||||
setText("stat-logs", "0");
|
||||
$("timeline").innerHTML = "";
|
||||
setText("command-preview", "");
|
||||
state.lastStatus = null;
|
||||
const pauseBtn = $("btn-pause");
|
||||
pauseBtn.disabled = true;
|
||||
pauseBtn.textContent = "暂停训练";
|
||||
chip("chip-converge", "判读", "等待数据", "gray");
|
||||
$("converge-notes").innerHTML = "";
|
||||
$("metric-grid").innerHTML = '<p class="hint">尚未启动训练,暂无参数。</p>';
|
||||
@@ -370,8 +379,14 @@ function renderProcess(job) {
|
||||
|
||||
renderTimeline(process);
|
||||
setText("command-preview", job.command ? `$ ${job.command.join(" ")}` : "");
|
||||
setFormLocked(job.status === "running" || job.status === "created" || job.status === "stopping");
|
||||
$("btn-stop").disabled = !(job.status === "running" || job.status === "created");
|
||||
state.lastStatus = job.status;
|
||||
setFormLocked(job.status === "running" || job.status === "created"
|
||||
|| job.status === "paused" || job.status === "stopping");
|
||||
$("btn-stop").disabled = !(job.status === "running" || job.status === "created"
|
||||
|| job.status === "paused");
|
||||
const pauseBtn = $("btn-pause");
|
||||
pauseBtn.disabled = !(job.status === "running" || job.status === "paused");
|
||||
pauseBtn.textContent = job.status === "paused" ? "继续训练" : "暂停训练";
|
||||
}
|
||||
|
||||
function renderTimeline(process) {
|
||||
@@ -427,8 +442,12 @@ function renderBanner(job) {
|
||||
}
|
||||
if (job.status === "failed") {
|
||||
showBanner("bad", `训练异常结束(退出码 <b>${job.returncode === null ? "?" : job.returncode}</b>):${escapeHtml(job.error || "见日志尾部")}。`);
|
||||
} else if (job.status === "paused") {
|
||||
showBanner("warn", "训练已暂停:进程与显存仍在,但停在 step 边界不再推进。点「继续训练」从暂停处恢复,或点「停止训练」保存当前模型后退出。");
|
||||
} else if (job.status === "stopped") {
|
||||
showBanner("warn", "已按请求停止训练(SIGTERM 独立进程组),脚本会在 step 边界保存当前模型与元数据。");
|
||||
showBanner("warn", job.forced_kill
|
||||
? "停止请求超时,已强制结束进程树;最后一段模型可能未保存,请检查 checkpoint 目录。"
|
||||
: "已按请求停止训练(控制文件协作式停止),脚本已在 step 边界保存当前模型与元数据。");
|
||||
} else if (job.status === "exited") {
|
||||
showBanner("ok", "训练与评估流程正常结束,模型与元数据已写入输出目录。");
|
||||
} else if (process.remaining_s !== null && process.remaining_s !== undefined
|
||||
@@ -439,6 +458,26 @@ function renderBanner(job) {
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 恢复
|
||||
function renderRecovery(job) {
|
||||
const recovery = job.recovery || {};
|
||||
const box = $("recovery-box");
|
||||
if (!box) return;
|
||||
if (!recovery.model || recovery.reason === "none") {
|
||||
box.classList.add("hidden");
|
||||
return;
|
||||
}
|
||||
box.classList.remove("hidden");
|
||||
const steps = recovery.timesteps === null || recovery.timesteps === undefined
|
||||
? "步数未知" : `${fmtInt(recovery.timesteps)} 步`;
|
||||
const when = recovery.updated_unix_s ? fmtClock(recovery.updated_unix_s) : "—";
|
||||
setText("recovery-text",
|
||||
`可恢复产物:${steps} · 来源 ${recovery.reason || "?"} · 更新 ${when} · ${recovery.model}`);
|
||||
const button = $("btn-recover");
|
||||
// 离线预览里的恢复信息是合成的,不允许真的去续训
|
||||
button.disabled = !recovery.can_resume || Boolean(state.preview);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 收敛
|
||||
function renderConvergence(job) {
|
||||
const convergence = job.convergence || {};
|
||||
@@ -679,6 +718,33 @@ function renderEpisodes(job) {
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 交互
|
||||
async function submitPayload(payload) {
|
||||
const err = $("form-error");
|
||||
err.textContent = "";
|
||||
if (state.saving) return false; // 训练中表单锁定,续训按钮同样不许重复提交
|
||||
if (payload.resume && !payload.model) {
|
||||
err.textContent = "续训需要填写模型输出路径";
|
||||
return false;
|
||||
}
|
||||
const res = await fetch("/api/train/start", {
|
||||
method: "POST", headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
const body = await res.json().catch(() => ({}));
|
||||
if (!res.ok) {
|
||||
err.textContent = body.error || `启动失败(HTTP ${res.status})`;
|
||||
return false;
|
||||
}
|
||||
state.preview = false;
|
||||
state.jobKey = null;
|
||||
state.metrics = [];
|
||||
state.metricCache = [];
|
||||
state.metricNext = 0;
|
||||
state.metricTotal = 0;
|
||||
await poll();
|
||||
return true;
|
||||
}
|
||||
|
||||
$("config-form").addEventListener("submit", async (event) => {
|
||||
event.preventDefault();
|
||||
if (state.saving) return;
|
||||
@@ -691,24 +757,43 @@ $("config-form").addEventListener("submit", async (event) => {
|
||||
for (const [key, value] of Object.entries(values)) {
|
||||
if (value !== null) payload[key] = value;
|
||||
}
|
||||
if (payload.resume && !payload.model) { err.textContent = "续训需要填写模型输出路径"; return; }
|
||||
const res = await fetch("/api/train/start", {
|
||||
method: "POST", headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(payload),
|
||||
await submitPayload(payload);
|
||||
});
|
||||
|
||||
$("btn-recover").addEventListener("click", async () => {
|
||||
const recovery = (state.lastJob && state.lastJob.recovery) || {};
|
||||
if (!recovery.can_resume || !recovery.model) return;
|
||||
if (!confirm(`用最近的可恢复产物续训?\n${recovery.model}\n`
|
||||
+ "当前表单里的 timesteps 会作为**追加**步数。")) return;
|
||||
// 同步回表单,用户随后可微调步数再手动点「开始训练」
|
||||
const modelField = $("cfg-model");
|
||||
if (modelField) modelField.value = recovery.model;
|
||||
const resumeField = $("cfg-resume");
|
||||
if (resumeField) resumeField.checked = true;
|
||||
const values = collectValues();
|
||||
values.model = recovery.model;
|
||||
values.resume = true;
|
||||
values.force = false; // --resume 与 --force 互斥
|
||||
await submitPayload(values);
|
||||
});
|
||||
|
||||
$("btn-pause").addEventListener("click", async () => {
|
||||
const resuming = state.lastStatus === "paused";
|
||||
const res = await fetch(resuming ? "/api/train/resume" : "/api/train/pause", {
|
||||
method: "POST", headers: { "Content-Type": "application/json" }, body: "{}",
|
||||
});
|
||||
const body = await res.json().catch(() => ({}));
|
||||
if (!res.ok) { err.textContent = body.error || `启动失败(HTTP ${res.status})`; return; }
|
||||
state.preview = false;
|
||||
state.jobKey = null;
|
||||
state.metrics = [];
|
||||
state.metricCache = [];
|
||||
state.metricNext = 0;
|
||||
state.metricTotal = 0;
|
||||
if (!res.ok) {
|
||||
showBanner("bad", `${resuming ? "继续" : "暂停"}失败:${escapeHtml(body.error || res.status)}`);
|
||||
} else if (!resuming && body.paused === false) {
|
||||
showBanner("warn", "当前没有可暂停的训练进程(可能已结束)。");
|
||||
}
|
||||
await poll();
|
||||
});
|
||||
|
||||
$("btn-stop").addEventListener("click", async () => {
|
||||
if (!confirm("确认停止当前训练?\n将向独立进程组发送 SIGTERM,脚本会先保存当前模型与元数据再退出。")) return;
|
||||
if (!confirm("确认停止当前训练?\n将通过控制文件请求优雅停止,脚本先保存当前模型与元数据再退出;"
|
||||
+ "若超时未响应则强制结束进程树。")) return;
|
||||
const res = await fetch("/api/train/stop", {
|
||||
method: "POST", headers: { "Content-Type": "application/json" }, body: "{}",
|
||||
});
|
||||
|
||||
@@ -30,6 +30,7 @@
|
||||
</select>
|
||||
</label>
|
||||
<button id="btn-preview" type="button" title="渲染示例快照,不启动任何训练进程">离线预览</button>
|
||||
<button id="btn-pause" type="button" disabled>暂停训练</button>
|
||||
<button id="btn-stop" class="danger" type="button" disabled>停止训练</button>
|
||||
</div>
|
||||
</header>
|
||||
@@ -37,6 +38,11 @@
|
||||
<main>
|
||||
<div id="banner" class="banner hidden"></div>
|
||||
|
||||
<div id="recovery-box" class="recovery hidden">
|
||||
<span class="hint" id="recovery-text">—</span>
|
||||
<button id="btn-recover" type="button" title="把最近一次完整落盘的产物作为 --resume 目标继续训练">用最近检查点续训</button>
|
||||
</div>
|
||||
|
||||
<!-- =============================================== 1. 执行进度与进程 -->
|
||||
<section class="panel" id="panel-process">
|
||||
<div class="panel-head">
|
||||
|
||||
@@ -64,6 +64,15 @@ button.danger:hover:enabled { background: rgba(248,113,113,.12); }
|
||||
.banner.warn { border-color: var(--warn); border-left-color: var(--warn); background: rgba(245,158,11,.08); }
|
||||
.banner b { font-family: var(--mono); }
|
||||
|
||||
/* 恢复条:显示最近一份完整落盘的产物,并提供一键续训 */
|
||||
.recovery {
|
||||
display: flex; align-items: center; justify-content: space-between; gap: 12px;
|
||||
border: 1px solid var(--cyan, #22d3ee); border-left: 4px solid var(--cyan, #22d3ee);
|
||||
background: rgba(34,211,238,.07); border-radius: 8px;
|
||||
padding: 8px 12px; margin: 10px 0 0; font-size: 13px;
|
||||
}
|
||||
.recovery .hint { overflow-wrap: anywhere; }
|
||||
|
||||
/* ------------------------------------------------------------ 面板 */
|
||||
.panel {
|
||||
border: 1px solid var(--line); border-radius: 12px; background: var(--panel);
|
||||
|
||||
Reference in New Issue
Block a user