1919810
This commit is contained in:
@@ -144,7 +144,8 @@ CONFIG_FIELDS: list[dict[str, Any]] = [
|
||||
{
|
||||
"key": "init", "kind": "text", "group": "paths", "optional": True,
|
||||
"label": "初始权重 --init", "default": None,
|
||||
"help": "留空=从零;填上一阶段 .zip 即课程迁移(步数计数重置)。",
|
||||
"help": "留空=从零;填上一阶段 .zip 即课程迁移。PPO 超参与步数计数沿用"
|
||||
"该模型(timesteps 按增量理解,进度条按增量显示)。",
|
||||
},
|
||||
{
|
||||
"key": "local_model", "kind": "text", "group": "paths", "optional": True,
|
||||
|
||||
@@ -739,7 +739,7 @@ _STATUS_VIEW = {
|
||||
"created": ("已创建", "gray", False, "命令已构造,等待 spawn"),
|
||||
"running": ("运行中", "ok", False, "子进程持有控制台输出与独立进程组"),
|
||||
"stopping": ("停止中", "warn", False, "已发送 SIGTERM,等待子进程退出"),
|
||||
"stopped": ("已手动停止", "warn", True, "收到停止信号后退出,模型可能未保存"),
|
||||
"stopped": ("已手动停止", "warn", True, "收到停止信号后退出;脚本会在 step 边界保存模型与元数据"),
|
||||
"exited": ("正常结束", "ok", True, "退出码 0:训练与评估流程走完"),
|
||||
"failed": ("异常结束", "bad", True, "非零退出码或启动失败,见日志尾部"),
|
||||
}
|
||||
@@ -812,15 +812,24 @@ def describe_process(snapshot: dict[str, Any]) -> dict[str, Any]:
|
||||
done_steps = value
|
||||
if done_steps == 0.0:
|
||||
done_steps = _finite_float(convergence.get("latest_timesteps")) or 0.0
|
||||
percent = (100.0 * done_steps / total_steps) if total_steps > 0 else None
|
||||
# init/resume(课程迁移)时 num_timesteps 从上一阶段继续,进度必须按本次
|
||||
# 增量算;start_timesteps 由训练脚本的进度 JSON 提供,缺省按 0 处理。
|
||||
start_steps = 0.0
|
||||
for point in metrics:
|
||||
value = _finite_float(point.get("start_timesteps"))
|
||||
if value is not None:
|
||||
start_steps = value
|
||||
break
|
||||
trained_steps = max(0.0, done_steps - start_steps)
|
||||
percent = (100.0 * trained_steps / total_steps) if total_steps > 0 else None
|
||||
if percent is not None:
|
||||
percent = max(0.0, min(100.0, percent))
|
||||
elapsed = float(snapshot.get("elapsed_s") or 0.0)
|
||||
rate = _latest_rate(metrics)
|
||||
remaining: float | None = None
|
||||
eta_ts: float | None = None
|
||||
if rate and rate > 0 and total_steps > 0 and done_steps < total_steps:
|
||||
remaining = (total_steps - done_steps) / rate
|
||||
if rate and rate > 0 and total_steps > 0 and trained_steps < total_steps:
|
||||
remaining = (total_steps - trained_steps) / rate
|
||||
eta_ts = float(snapshot.get("finished_at") or time.time()) + remaining
|
||||
if not terminal and remaining is None:
|
||||
cap = _finite_float(config.get("max_wall_s")) or 0.0
|
||||
@@ -835,6 +844,8 @@ def describe_process(snapshot: dict[str, Any]) -> dict[str, Any]:
|
||||
"timeline": timeline,
|
||||
"percent": percent,
|
||||
"done_steps": done_steps,
|
||||
"start_steps": start_steps,
|
||||
"trained_steps": trained_steps,
|
||||
"total_steps": total_steps,
|
||||
"elapsed_s": round(elapsed, 3),
|
||||
"rate_per_s": rate,
|
||||
|
||||
@@ -347,7 +347,7 @@ function renderProcess(job) {
|
||||
chip("chip-progress", "进度",
|
||||
percent === null || percent === undefined ? "—" : `${percent.toFixed(1)}%`,
|
||||
percent !== null && percent >= 100 ? "ok" : "cyan");
|
||||
setText("steps-done", fmtInt(process.done_steps));
|
||||
setText("steps-done", fmtInt(process.trained_steps ?? process.done_steps));
|
||||
setText("steps-total", fmtInt(process.total_steps));
|
||||
setText("progress-percent", percent === null || percent === undefined ? "—" : `${percent.toFixed(2)}%`);
|
||||
const fill = $("bar-fill");
|
||||
@@ -428,7 +428,7 @@ function renderBanner(job) {
|
||||
if (job.status === "failed") {
|
||||
showBanner("bad", `训练异常结束(退出码 <b>${job.returncode === null ? "?" : job.returncode}</b>):${escapeHtml(job.error || "见日志尾部")}。`);
|
||||
} else if (job.status === "stopped") {
|
||||
showBanner("warn", "已按请求停止训练(SIGTERM 独立进程组),本轮模型可能未保存。");
|
||||
showBanner("warn", "已按请求停止训练(SIGTERM 独立进程组),脚本会在 step 边界保存当前模型与元数据。");
|
||||
} else if (job.status === "exited") {
|
||||
showBanner("ok", "训练与评估流程正常结束,模型与元数据已写入输出目录。");
|
||||
} else if (process.remaining_s !== null && process.remaining_s !== undefined
|
||||
@@ -708,7 +708,7 @@ $("config-form").addEventListener("submit", async (event) => {
|
||||
});
|
||||
|
||||
$("btn-stop").addEventListener("click", async () => {
|
||||
if (!confirm("确认停止当前训练?\n将向独立进程组发送 SIGTERM,本轮模型可能未保存。")) return;
|
||||
if (!confirm("确认停止当前训练?\n将向独立进程组发送 SIGTERM,脚本会先保存当前模型与元数据再退出。")) return;
|
||||
const res = await fetch("/api/train/stop", {
|
||||
method: "POST", headers: { "Content-Type": "application/json" }, body: "{}",
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user