This commit is contained in:
Horldsence
2026-09-11 23:07:42 +08:00
parent 5c96699b70
commit 36f354736c
8 changed files with 522 additions and 23 deletions
+2 -1
View File
@@ -144,7 +144,8 @@ CONFIG_FIELDS: list[dict[str, Any]] = [
{
"key": "init", "kind": "text", "group": "paths", "optional": True,
"label": "初始权重 --init", "default": None,
"help": "留空=从零;填上一阶段 .zip 即课程迁移(步数计数重置)。",
"help": "留空=从零;填上一阶段 .zip 即课程迁移。PPO 超参与步数计数沿用"
"该模型(timesteps 按增量理解,进度条按增量显示)。",
},
{
"key": "local_model", "kind": "text", "group": "paths", "optional": True,
+15 -4
View File
@@ -739,7 +739,7 @@ _STATUS_VIEW = {
"created": ("已创建", "gray", False, "命令已构造,等待 spawn"),
"running": ("运行中", "ok", False, "子进程持有控制台输出与独立进程组"),
"stopping": ("停止中", "warn", False, "已发送 SIGTERM,等待子进程退出"),
"stopped": ("已手动停止", "warn", True, "收到停止信号后退出,模型可能未保存"),
"stopped": ("已手动停止", "warn", True, "收到停止信号后退出;脚本会在 step 边界保存模型与元数据"),
"exited": ("正常结束", "ok", True, "退出码 0:训练与评估流程走完"),
"failed": ("异常结束", "bad", True, "非零退出码或启动失败,见日志尾部"),
}
@@ -812,15 +812,24 @@ def describe_process(snapshot: dict[str, Any]) -> dict[str, Any]:
done_steps = value
if done_steps == 0.0:
done_steps = _finite_float(convergence.get("latest_timesteps")) or 0.0
percent = (100.0 * done_steps / total_steps) if total_steps > 0 else None
# init/resume(课程迁移)时 num_timesteps 从上一阶段继续,进度必须按本次
# 增量算;start_timesteps 由训练脚本的进度 JSON 提供,缺省按 0 处理。
start_steps = 0.0
for point in metrics:
value = _finite_float(point.get("start_timesteps"))
if value is not None:
start_steps = value
break
trained_steps = max(0.0, done_steps - start_steps)
percent = (100.0 * trained_steps / total_steps) if total_steps > 0 else None
if percent is not None:
percent = max(0.0, min(100.0, percent))
elapsed = float(snapshot.get("elapsed_s") or 0.0)
rate = _latest_rate(metrics)
remaining: float | None = None
eta_ts: float | None = None
if rate and rate > 0 and total_steps > 0 and done_steps < total_steps:
remaining = (total_steps - done_steps) / rate
if rate and rate > 0 and total_steps > 0 and trained_steps < total_steps:
remaining = (total_steps - trained_steps) / rate
eta_ts = float(snapshot.get("finished_at") or time.time()) + remaining
if not terminal and remaining is None:
cap = _finite_float(config.get("max_wall_s")) or 0.0
@@ -835,6 +844,8 @@ def describe_process(snapshot: dict[str, Any]) -> dict[str, Any]:
"timeline": timeline,
"percent": percent,
"done_steps": done_steps,
"start_steps": start_steps,
"trained_steps": trained_steps,
"total_steps": total_steps,
"elapsed_s": round(elapsed, 3),
"rate_per_s": rate,
+3 -3
View File
@@ -347,7 +347,7 @@ function renderProcess(job) {
chip("chip-progress", "进度",
percent === null || percent === undefined ? "—" : `${percent.toFixed(1)}%`,
percent !== null && percent >= 100 ? "ok" : "cyan");
setText("steps-done", fmtInt(process.done_steps));
setText("steps-done", fmtInt(process.trained_steps ?? process.done_steps));
setText("steps-total", fmtInt(process.total_steps));
setText("progress-percent", percent === null || percent === undefined ? "—" : `${percent.toFixed(2)}%`);
const fill = $("bar-fill");
@@ -428,7 +428,7 @@ function renderBanner(job) {
if (job.status === "failed") {
showBanner("bad", `训练异常结束(退出码 <b>${job.returncode === null ? "?" : job.returncode}</b>):${escapeHtml(job.error || "见日志尾部")}。`);
} else if (job.status === "stopped") {
showBanner("warn", "已按请求停止训练(SIGTERM 独立进程组),本轮模型可能未保存。");
showBanner("warn", "已按请求停止训练(SIGTERM 独立进程组),脚本会在 step 边界保存当前模型与元数据。");
} else if (job.status === "exited") {
showBanner("ok", "训练与评估流程正常结束,模型与元数据已写入输出目录。");
} else if (process.remaining_s !== null && process.remaining_s !== undefined
@@ -708,7 +708,7 @@ $("config-form").addEventListener("submit", async (event) => {
});
$("btn-stop").addEventListener("click", async () => {
if (!confirm("确认停止当前训练?\n将向独立进程组发送 SIGTERM,本轮模型可能未保存。")) return;
if (!confirm("确认停止当前训练?\n将向独立进程组发送 SIGTERM,脚本会先保存当前模型与元数据再退出。")) return;
const res = await fetch("/api/train/stop", {
method: "POST", headers: { "Content-Type": "application/json" }, body: "{}",
});