holy fuking shit

2026-09-12 01:56:34 +08:00
parent 36f354736c
commit d975309a81
83 changed files with 2748 additions and 896 deletions
@@ -6,11 +6,16 @@
 - SB3 表格解析必须给每一行打上 ``total_timesteps`` 横轴（曾把表格行当独立
  点，曲线上出现重复的幽灵点）；
 - ``metric_offset``/``metric_end`` 游标契约（前端增量拉取依赖它）；
- 用一段桩训练脚本真起子进程，验证启动/轮询/停止/退出码全链路。
+- 用一段桩训练脚本真起子进程，验证启动/轮询/暂停/继续/停止/退出码全链路。
 """

+# pytest 用参数名注入 fixture：`stub_repo`/`client` 这类参数在函数体里不直接
+# 引用也是正常用法（它们的作用是 setup/teardown），这里统一放行。
+# pylint: disable=unused-argument
+
 from __future__ import annotations

+import dataclasses
 import http.client
 import json
 import sys
@@ -22,6 +27,7 @@ import pytest

 pytest.importorskip("mcm26b")

+from mcm26b import process_control as pc
 from mcm26b.training_ui import demo, server as td
 from mcm26b.training_ui.schema import (CONFIG_FIELDS, config_schema,
                                       preset_payloads)
@@ -32,7 +38,7 @@ REPO = Path(__file__).resolve().parents[1]
 # 最后打印一行元数据。全部行为由参数控制，不依赖 torch/SB3。
 STUB_SCRIPT = r'''
 """训练前端测试桩：模拟 SB3 verbose=1 表格 + --progress-json 输出。"""
-import argparse, json, signal, sys, time
+import argparse, json, os, signal, sys, time
 from pathlib import Path

 parser = argparse.ArgumentParser()
@@ -41,12 +47,17 @@ parser.add_argument("--stage", type=int, default=1)
 parser.add_argument("--timesteps", type=int, default=96)
 parser.add_argument("--model", default="out/model.zip")
 parser.add_argument("--rollout-steps", type=int, default=32)
-parser.add_argument("--sleep", type=float, default=0.0)
+# 默认 sleep 可由环境变量控制：HTTP 端到端用例无法直接追加命令行参数，
+# 用小 sleep 拉长运行窗口，才能稳定地测"暂停/继续"。
+parser.add_argument("--sleep", type=float,
+                    default=float(os.environ.get("STUB_TRAIN_SLEEP", "0")))
 parser.add_argument("--lines", type=int, default=4)
 parser.add_argument("--table-first", action="store_true")
 parser.add_argument("--progress-json", action="store_true")
 parser.add_argument("--crash", action="store_true")
 parser.add_argument("--ignore-sigterm", action="store_true")
+parser.add_argument("--ignore-control", action="store_true")
+parser.add_argument("--control-file", default=None)
 args, _extra = parser.parse_known_args()

 model = Path(args.model)
@@ -65,6 +76,49 @@ print("NORMAL-LOG-LINE")
 sys.stdout.flush()


+def read_control():
+    if not args.control_file or args.ignore_control:
+        return {}
+    try:
+        with open(args.control_file, encoding="utf-8") as handle:
+            return json.load(handle)
+    except Exception:
+        return {}
+
+
+def control_pause_or_stop(step):
+    """返回 True 表示应退出；暂停时阻塞到继续或停止（模拟协作式暂停）。"""
+    state = read_control()
+    if state.get("stop"):
+        print(json.dumps({"event": "stop_requested", "signal": "control_file"}), flush=True)
+        return True
+    if state.get("paused"):
+        print(json.dumps({"event": "paused", "timesteps": step}), flush=True)
+        while True:
+            time.sleep(0.05)
+            state = read_control()
+            if state.get("stop"):
+                print(json.dumps({"event": "stop_requested", "signal": "control_file"}),
+                      flush=True)
+                return True
+            if not state.get("paused"):
+                break
+        print(json.dumps({"event": "resumed", "timesteps": step}), flush=True)
+    return False
+
+
+def sleep_with_control(seconds, step):
+    """把长 sleep 切成 0.1s 片，保证停止/暂停请求能被及时响应。"""
+    deadline = time.monotonic() + max(0.0, seconds)
+    while True:
+        if control_pause_or_stop(step):
+            raise SystemExit(0)
+        remaining = deadline - time.monotonic()
+        if remaining <= 0:
+            return
+        time.sleep(min(0.1, remaining))
+
+
 def render_table(step, fps):
    return (
        "-----------------------------\n"
@@ -72,16 +126,15 @@ def render_table(step, fps):
        "|    ep_len_mean     | 9.67     |\n"
        "|    ep_rew_mean     | -3.24    |\n"
        "| time/              |          |\n"
-        "|    fps             | %-8d |\n"
-        "|    total_timesteps | %-8d |\n"
-        "-----------------------------\n" % (fps, step)
+        f"|    fps             | {fps:<8d} |\n"
+        f"|    total_timesteps | {step:<8d} |\n"
+        "-----------------------------\n"
    )


 for index in range(args.lines):
-    if args.sleep:
-        time.sleep(args.sleep)
    step = 32 * (index + 1)
+    sleep_with_control(args.sleep, step)
    fps = 17 + index
    progress = {
        "event": "progress", "timesteps": step, "n_updates": index + 1,
@@ -153,7 +206,7 @@ def test_schema_matches_train_config_defaults():
        assert field["default"] == getattr(defaults, field["key"]), \
            f"{field['key']} 默认值与后端不一致"
    exposed = {field["key"] for field in schema["fields"]}
-    missing = set(defaults.__dataclass_fields__) - exposed
+    missing = {field.name for field in dataclasses.fields(defaults)} - exposed
    assert not missing, f"TrainConfig 字段未暴露给界面: {sorted(missing)}"
    groups = {group["key"] for group in schema["groups"]}
    for field in schema["fields"]:
@@ -227,9 +280,11 @@ class _FakeJob:
        self._last_table_steps = None
        self._last_json_steps = None
        self._last_monitor_sig = None
+        self.status = "running"
        self.config = make_config()

    _on_line = td.TrainingJob._on_line
+    _sync_control_events = td.TrainingJob._sync_control_events
    _parse_progress_json = td.TrainingJob._parse_progress_json
    _parse_sb3_table = td.TrainingJob._parse_sb3_table
    _flush_table = td.TrainingJob._flush_table
@@ -274,9 +329,9 @@ def test_sb3_table_with_separators_emits_one_point_per_table():
    for index, steps in enumerate((32, 64, 96)):
        job._on_line("-----------------------------")
        job._on_line("| rollout/           |          |")
-        job._on_line("|    ep_rew_mean     | %.2f    |" % (-3.0 + index))
+        job._on_line(f"|    ep_rew_mean     | {-3.0 + index:.2f}    |")
        job._on_line("| time/              |          |")
-        job._on_line("|    total_timesteps | %d       |" % steps)
+        job._on_line(f"|    total_timesteps | {steps}       |")
    job._on_line("-----------------------------")
    job._flush_table()
    assert [point["timesteps"] for point in job.metrics] == [32, 64, 96], list(job.metrics)
@@ -476,7 +531,7 @@ def test_job_lifecycle_with_stub_script(stub_repo):
 def test_job_reports_nonzero_exit_code(stub_repo):
    config = make_config(model="output/hrl/ui/test/crash/q3.zip")
    job = td.TrainingJob(config)
-    job.command = td.build_command(config) + ["--crash"]
+    job.command = list(job.command) + ["--crash"]
    job.start()
    assert wait_until(lambda: job.status in ("exited", "failed"), timeout=30)
    assert job.status == "failed"
@@ -484,15 +539,200 @@ def test_job_reports_nonzero_exit_code(stub_repo):
    assert job.snapshot(log_tail=5)["process"]["tone"] == "bad"


-def test_stop_sends_sigterm_and_marks_stopped(stub_repo):
+def test_job_command_carries_control_file(stub_repo):
+    """控制文件路径必须随命令下发，否则暂停/停止退化成超时强杀。"""
+    config = make_config(model="output/hrl/ui/test/control/q3.zip")
+    job = td.TrainingJob(config, control_dir=Path(stub_repo).parent)
+    assert "--control-file" in job.command
+    assert str(job.control_path) in job.command
+    assert job.control_path.parent == Path(stub_repo).parent
+
+
+def test_stop_gracefully_marks_stopped_without_forced_kill(stub_repo):
    config = make_config(model="output/hrl/ui/test/stop/q3.zip")
    job = td.TrainingJob(config)
-    job.command = td.build_command(config) + ["--sleep", "5", "--lines", "50"]
+    job.command = list(job.command) + ["--sleep", "5", "--lines", "50"]
    job.start()
    assert wait_until(lambda: job.status == "running", timeout=10)
    assert job.stop() is True
    assert wait_until(lambda: job.status in ("stopped", "failed"), timeout=15)
    assert job.status == "stopped"
+    # 桩脚本读控制文件后主动退出：不应走到强杀分支
+    assert job.forced_kill is False
+
+
+def test_pause_and_resume_via_control_file(stub_repo):
+    config = make_config(model="output/hrl/ui/test/pause/q3.zip")
+    job = td.TrainingJob(config)
+    job.command = list(job.command) + ["--sleep", "0.2", "--lines", "60"]
+    job.start()
+    assert wait_until(lambda: job.status == "running", timeout=10)
+    assert job.pause() is True
+    # 训练脚本回显 paused 事件；状态由 _sync_control_events 校正
+    assert wait_until(
+        lambda: any('"event": "paused"' in line for line in job.snapshot()["log_lines"]),
+        timeout=10), "训练脚本未回显 paused 事件"
+    assert job.status == "paused"
+    paused_metrics = len(job.snapshot()["metrics"])
+    time.sleep(0.6)
+    assert len(job.snapshot()["metrics"]) <= paused_metrics + 1, "暂停后仍在产出指标"
+    assert job.resume() is True
+    assert wait_until(
+        lambda: any('"event": "resumed"' in line for line in job.snapshot()["log_lines"]),
+        timeout=10), "训练脚本未回显 resumed 事件"
+    assert wait_until(lambda: job.status == "running", timeout=5)
+    assert job.stop() is True
+    assert wait_until(lambda: job.status in ("stopped", "failed"), timeout=15)
+    assert job.status == "stopped"
+
+
+def test_stop_forced_kill_when_control_ignored(stub_repo, monkeypatch):
+    """训练进程不读控制文件时必须超时强杀，并留下恢复指引。"""
+    monkeypatch.setattr(td, "STOP_GRACE_S", 0.5)
+    config = make_config(model="output/hrl/ui/test/stubborn/q3.zip")
+    job = td.TrainingJob(config)
+    job.command = list(job.command) + ["--ignore-control", "--sleep", "5", "--lines", "50"]
+    job.start()
+    assert wait_until(lambda: job.status == "running", timeout=10)
+    # 造一份"磁盘上已有滚动恢复点"的现场，强杀后应被记录进恢复指引
+    ckpt_dir = job.config.model_path.parent / "checkpoints"
+    ckpt_dir.mkdir(parents=True, exist_ok=True)
+    latest = ckpt_dir / "q3_ckpt_latest.zip"
+    latest.write_bytes(b"PK\x03\x04fake")
+    assert job.stop() is True
+    assert wait_until(lambda: job.status in ("stopped", "failed"), timeout=20)
+    assert job.forced_kill is True
+    assert job.snapshot(log_tail=5)["forced_kill"] is True
+    marker = json.loads(
+        (job.config.model_path.parent / "forced_stop.json").read_text(encoding="utf-8"))
+    assert marker["event"] == "forced_stop"
+    assert marker["reason"] == "grace_timeout"
+    assert marker["recovery"]["model"] == str(latest)
+    assert marker["recovery"]["can_resume"] is True
+
+
+def test_discover_recovery_prefers_resume_hint(tmp_path):
+    run_dir = tmp_path / "run"
+    (run_dir / "checkpoints").mkdir(parents=True)
+    latest = run_dir / "checkpoints" / "q3_ckpt_latest.zip"
+    latest.write_bytes(b"PK\x03\x04fake")
+    final = run_dir / "q3.zip"
+    final.write_bytes(b"PK\x03\x04fake")
+    (run_dir / "resume.json").write_text(json.dumps({
+        "model": str(latest), "timesteps": 128, "reason": "interval",
+        "created_unix_s": 5.0}), encoding="utf-8")
+    state = td.discover_recovery(run_dir, final)
+    assert state["can_resume"] is True
+    assert state["model"] == str(latest)
+    assert state["timesteps"] == 128
+    assert state["source"] == "resume.json"
+
+
+def test_discover_recovery_falls_back_to_scan_on_dangling_hint(tmp_path):
+    run_dir = tmp_path / "run"
+    (run_dir / "checkpoints").mkdir(parents=True)
+    latest = run_dir / "checkpoints" / "q3_ckpt_latest.zip"
+    latest.write_bytes(b"PK\x03\x04fake")
+    # 悬空指针（目标文件已不在）必须回退到扫描，而不是给出一个打不开的路径
+    (run_dir / "resume.json").write_text(
+        json.dumps({"model": str(run_dir / "gone.zip")}), encoding="utf-8")
+    state = td.discover_recovery(run_dir, run_dir / "q3.zip")
+    assert state["can_resume"] is True
+    assert state["model"] == str(latest)
+    assert state["source"] == "scan"
+
+
+def test_discover_recovery_with_nothing_on_disk(tmp_path):
+    state = td.discover_recovery(tmp_path, tmp_path / "q3.zip")
+    assert state["can_resume"] is False and state["model"] is None
+    assert state["reason"] == "none"
+
+
+def test_command_carries_checkpoint_and_recovery_flags():
+    config = make_config(model="output/x/q3.zip", recovery_every=100,
+                         checkpoint_every=500, checkpoint_keep=2)
+    joined = " ".join(td.build_command(config))
+    assert "--recovery-every 100" in joined
+    assert "--checkpoint-every 500" in joined
+    assert "--checkpoint-keep 2" in joined
+
+
+def test_config_rejects_bad_recovery_settings():
+    with pytest.raises(ValueError):
+        make_config(recovery_every=-1)
+    with pytest.raises(ValueError):
+        make_config(checkpoint_every=-1)
+    with pytest.raises(ValueError):
+        make_config(checkpoint_keep=0)
+
+
+def test_pause_resume_return_false_without_live_process(stub_repo):
+    config = make_config(model="output/hrl/ui/test/nolive/q3.zip")
+    job = td.TrainingJob(config)
+    assert job.pause() is False and job.resume() is False
+    job.start()
+    assert wait_until(lambda: job.status in ("exited", "failed"), timeout=30)
+    assert job.pause() is False and job.resume() is False
+
+
+# ---------------------------------------------------- 跨平台进程控制
+def test_control_file_roundtrip_and_corruption(tmp_path):
+    path = tmp_path / "control.json"
+    assert pc.read_control(path) == pc.ControlState()      # 缺失 = 无指令
+    pc.write_control(path, paused=True)
+    assert pc.read_control(path).paused is True
+    pc.write_control(path, stop=True)
+    state = pc.read_control(path)
+    assert state.stop is True and state.paused is False
+    path.write_text("{ 半截 JSON", encoding="utf-8")
+    assert pc.read_control(path) == pc.ControlState()      # 损坏 = 无指令
+
+
+def test_spawn_kwargs_isolates_child_process_group(monkeypatch):
+    monkeypatch.setattr(pc, "IS_WINDOWS", False)
+    assert pc.spawn_kwargs() == {"start_new_session": True}
+    monkeypatch.setattr(pc, "IS_WINDOWS", True)
+    kwargs = pc.spawn_kwargs()
+    # Windows 上 start_new_session 被 CPython 静默忽略，绝不能再用它
+    assert "start_new_session" not in kwargs
+    if kwargs:
+        assert kwargs["creationflags"] == pc._CREATE_NEW_PROCESS_GROUP
+
+
+def test_hard_kill_on_windows_uses_taskkill(monkeypatch):
+    commands = []
+
+    class _FakeProcess:
+        pid = 4242
+
+        def kill(self):  # pragma: no cover - Windows 分支不应走到这里
+            commands.append("Popen.kill")
+
+    monkeypatch.setattr(pc, "IS_WINDOWS", True)
+    monkeypatch.setattr(pc.subprocess, "run",
+                        lambda argv, **kwargs: commands.append(argv))
+    pc._hard_kill_tree(_FakeProcess())
+    assert commands and commands[0][0] == "taskkill"
+    assert "/PID" in commands[0] and "4242" in commands[0]
+
+
+def test_hard_kill_without_killpg_does_not_raise(monkeypatch):
+    """回归：Windows 上 os 没有 killpg/getpgid，旧代码会抛 AttributeError。"""
+    killed = []
+
+    class _FakeProcess:
+        pid = 4242
+
+        def kill(self):
+            killed.append(True)
+
+    class _PosixlessOS:
+        """模拟 Windows：只有 getpid 之类，没有 killpg/getpgid。"""
+
+    monkeypatch.setattr(pc, "IS_WINDOWS", False)
+    monkeypatch.setattr(pc, "os", _PosixlessOS())
+    pc._hard_kill_tree(_FakeProcess())
+    assert killed == [True]


 def test_reader_flushes_truncated_table_on_exit(stub_repo):
@@ -628,8 +868,42 @@ def test_http_start_poll_stop(client):

    status, body = call(port, "POST", "/api/train/stop", {})
    assert status == 200 and body["stopped"] is True
+    # 等训练进程真正退出后再点一次：应返回 False 而不是报错
+    assert wait_until(
+        lambda: call(port, "GET", "/api/train/state?log_tail=5")[1]["job"]["status"]
+        in ("stopped", "failed"), timeout=15)
    status, body = call(port, "POST", "/api/train/stop", {})
-    assert status == 200 and body["stopped"] is False   # 已结束，返回 False 而不是报错
+    assert status == 200 and body["stopped"] is False
+
+
+def test_http_pause_resume_and_state(client, monkeypatch):
+    monkeypatch.setenv("STUB_TRAIN_SLEEP", "0.6")
+    _ui, port = client
+    status, body = call(port, "POST", "/api/train/start",
+                        {"model": "output/hrl/ui/test/http-pause/q3.zip",
+                         "timesteps": 4096, "rollout_steps": 32, "batch_size": 32})
+    assert status == 200
+
+    def status_now():
+        return call(port, "GET", "/api/train/state?log_tail=20")[1]["job"]["status"]
+
+    def is_running():
+        return status_now() == "running"
+
+    assert wait_until(is_running, timeout=10)
+    status, body = call(port, "POST", "/api/train/pause", {})
+    assert status == 200 and body["paused"] is True
+    assert wait_until(lambda: status_now() == "paused", timeout=10), status_now()
+    # 暂停中仍视为"有任务在跑"：不允许再开一局
+    status, body = call(port, "POST", "/api/train/start",
+                        {"model": "output/hrl/ui/test/http-pause2/q3.zip"})
+    assert status == 409 and "已有训练任务" in body["error"]
+    status, body = call(port, "POST", "/api/train/resume", {})
+    assert status == 200 and body["resumed"] is True
+    assert wait_until(is_running, timeout=10), status_now()
+    status, body = call(port, "POST", "/api/train/stop", {})
+    assert status == 200 and body["stopped"] is True
+    assert wait_until(lambda: status_now() in ("stopped", "failed"), timeout=15)


 def test_http_rejects_second_job_while_running(client):
@@ -661,7 +935,7 @@ def test_http_binds_loopback_only(client):


 def test_port_conflict_reports_actionable_error(client):
-    ui, port = client
+    _ui, port = client
    with pytest.raises(OSError) as excinfo:
        td.TrainingUI(ui_port=port)
    assert "ui-port" in str(excinfo.value)