sloray/mcm-26-b
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

holy fuking shit

Browse Source
This commit is contained in:
Horldsence
2026-09-12 01:56:34 +08:00
parent 36f354736c
commit d975309a81
83 changed files with 2748 additions and 896 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/benchmark_hrl.py
+2 -3
View File
@@ -26,14 +26,13 @@ from __future__ import annotations
import argparse
import csv
import json
import math
import statistics
import sys
import time
from collections import defaultdict
from dataclasses import asdict, dataclass
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Callable
from typing import Any
from mcm26b.config import ProblemConfig
from mcm26b.harness import LocalSimTransport, mixed_scene, random_omni_scene
scripts/benchmark_q1.py
+10 -6
View File
@@ -39,17 +39,21 @@ def main():
for margin in [0, 0.005]:
ms = [replace(m, half_angle_deg=m.half_angle_deg+margin) for m in observations]
result, duration = timed(solve, ms, 31)
cert = result['clearance_certificate']
sensitivity.append(dict(additional_half_angle_deg=margin, solve_ms=duration,
diameter_m=result['diameter_m'],
mec_radius_m=result['minimum_enclosing_circle']['radius_m'],
diameter_circle_excess_m=result['diameter_circle']['max_excess_m'],
clearance_margin_m=result['clearance_certificate']['remaining_margin_m']))
report = dict(python=sys.version.split()[0], timing_method='median perf_counter; warm local process',
timing_scope='MEC only; not whole solver speedup or simulator/network performance',
circle_benchmarks=timings, practice_sensitivity=sensitivity,
q2_first_bearing_certificate=first_bearing_candidate_disk(Measurement(0, 0, 0, 1.005)))
clearance_margin_m=cert['remaining_margin_m']))
report = dict(
python=sys.version.split()[0],
timing_method='median perf_counter; warm local process',
timing_scope='MEC only; not whole solver speedup or simulator/network performance',
circle_benchmarks=timings, practice_sensitivity=sensitivity,
q2_first_bearing_certificate=first_bearing_candidate_disk(Measurement(0, 0, 0, 1.005)))
output = ROOT/'output/q1/optimization_report.json'
output.write_text(json.dumps(report, ensure_ascii=False, indent=2, allow_nan=False)+'\n', encoding='utf-8')
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2, allow_nan=False)+'\n', encoding='utf-8')
print(json.dumps(report, ensure_ascii=False, indent=2))
scripts/benchmark_q3_revision.py
+12 -6
View File
@@ -48,19 +48,22 @@ def main():
for p in sorted(Path('src/mcm26b').rglob('*.py'))}
if args.baseline:
for name in ('baseline_bayes.py', 'baseline_config.json'):
source_sha256[str(args.out/name)] = hashlib.sha256((args.out/name).read_bytes()).hexdigest()
archive = args.out/name
source_sha256[str(archive)] = hashlib.sha256(archive.read_bytes()).hexdigest()
scenes = []
if args.suite == 'development':
for count in (10, 16):
for seed in range(6):
scenes.append((f'random-{count}-{seed}', random_omni_scene(seed, count), 20260910 + seed))
scenes.append((f'random-{count}-{seed}', random_omni_scene(seed, count),
20260910 + seed))
edge = [Source(c+1, 1800*math.cos(math.pi/6+2*math.pi*c/count),
1800*math.sin(math.pi/6+2*math.pi*c/count), 1000) for c in range(count)]
scenes.append((f'boundary-{count}', edge, 20260910))
else:
for count in (10, 13, 16):
for seed in range(100, 106):
scenes.append((f'holdout-{count}-{seed}', random_omni_scene(seed, count), 20260910+seed))
scenes.append((f'holdout-{count}-{seed}', random_omni_scene(seed, count),
20260910+seed))
rows = []
for name, sources, error_seed in scenes:
sim = LocalSimulator(sources, robot_id='BENCH', config=config, error_seed=error_seed)
@@ -81,12 +84,15 @@ def main():
sources=[asdict(s) for s in sources], error_seed=error_seed,
summary=summary, actions=actions), ensure_ascii=False), encoding='utf-8')
print(f'{name}: valid={valid} T={row["virtual_s"]:.1f} M={row["measures"]}', flush=True)
result = dict(config=asdict(config), source='local_simulator_only', baseline=args.baseline, rows=rows,
mean_s=mean(r['virtual_s'] for r in rows), median_s=median(r['virtual_s'] for r in rows),
result = dict(config=asdict(config), source='local_simulator_only',
baseline=args.baseline, rows=rows,
mean_s=mean(r['virtual_s'] for r in rows),
median_s=median(r['virtual_s'] for r in rows),
max_s=max(r['virtual_s'] for r in rows), valid=sum(r['valid'] for r in rows),
under_4000=sum(r['virtual_s'] < 4000 for r in rows),
source_sha256=source_sha256)
(folder/'results.json').write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8')
(folder/'results.json').write_text(
json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8')
print({k:result[k] for k in ('mean_s','median_s','max_s','valid','under_4000')}, flush=True)
scripts/benchmark_q3_search.py
+14 -9
View File
@@ -14,14 +14,11 @@ from pathlib import Path
from statistics import mean, median
from time import perf_counter
from mcm26b.belief import ChannelStatus # noqa: F401 (兼容旧 import 路径)
from mcm26b.config import ProblemConfig
from mcm26b.coverage import CoverageLedger # noqa: F401
from mcm26b.planner import OmnidirectionalLoop, clear_certificate # noqa: F401
from mcm26b.planner import OmnidirectionalLoop
from mcm26b.planner.seven import (SevenBatch, SevenImmediate, SevenSweep,
RING_RADIUS_M, seven_points)
from mcm26b.protocol.client import SimulatorClient
from mcm26b.protocol.rules import NEAR # noqa: F401
from mcm26b.harness import LocalSimTransport, LocalSimulator
from referee import Source
from mcm26b.harness import random_omni_scene
@@ -70,7 +67,8 @@ def run_case(name, sources, error_seed, out):
"actions_file": str(log_path.relative_to(out)),
"actions_sha256": hashlib.sha256(log_path.read_bytes()).hexdigest()}
rows.append(row)
print(f"{name} {label}: all_cleared={row['all_cleared']} T={row['virtual_time_s']:.1f}s M={row['measures']}", flush=True)
print(f"{name} {label}: all_cleared={row['all_cleared']} "
f"T={row['virtual_time_s']:.1f}s M={row['measures']}", flush=True)
return rows
@@ -96,7 +94,8 @@ def geometry_check():
"sampled_max_distance_m": worst[0], "sampled_worst_point": worst[1],
"samples": 181*721,
"passed": worst[0] <= analytic+1e-8 and analytic < field.r_eff_min_m,
"note": "Grid samples check the derivation; the sector proof establishes continuous coverage."}
"note": ("Grid samples check the derivation; "
"the sector proof establishes continuous coverage.")}
def main():
@@ -137,10 +136,16 @@ def main():
result = {"source": "local_simulator_only", "production_algorithm_modified": False,
"seeds": args.seeds, "config": asdict(ProblemConfig()),
"geometry": geometry_check(), "aggregates": aggregates, "runs": rows,
"limitations": "Same scene and fixed per-position error field for each variant; synthetic data, not an official improvement estimate. Only search/scheduling is changed; no posterior or localizer redesign."}
(args.out/"benchmark.json").write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
"limitations": (
"Same scene and fixed per-position error field for each variant; "
"synthetic data, not an official improvement estimate. "
"Only search/scheduling is changed; no posterior or localizer redesign.")}
(args.out/"benchmark.json").write_text(
json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
print(json.dumps(aggregates, ensure_ascii=False, indent=2))
return 0 if result["geometry"]["passed"] and all(a["valid_runs"] == a["runs"] for a in aggregates.values()) else 1
passed = (result["geometry"]["passed"]
and all(a["valid_runs"] == a["runs"] for a in aggregates.values()))
return 0 if passed else 1
if __name__ == "__main__":
scripts/benchmark_rl.py
+2 -1
View File
@@ -95,7 +95,8 @@ def main(argv=None) -> int:
cleared = all(row["rl"]["all_cleared"] and row[baseline]["all_cleared"]
for row in rows)
valid = all(part['completed'] and part['all_cleared'] and part['correct_absence']
and part['normal_exit'] and not part['anomalies'] and not part['timing_mismatches']
and part['normal_exit'] and not part['anomalies']
and not part['timing_mismatches']
for row in rows for part in (row[baseline],row['rl']))
report[key + "_summary"] = {
"mean_virtual_s": {
scripts/build_q1_demo.py
+13 -5
View File
@@ -53,15 +53,23 @@ def main():
data={**case,'measurements':[asdict(m) for m in ms],'frames':frames}
dataset.append(data)
if case['source']=='synthetic':
(examples/(case['id']+'.json')).write_text(json.dumps({k:v for k,v in data.items() if k!='frames'},ensure_ascii=False,indent=2),encoding='utf-8')
(results/(case['id']+'_result.json')).write_text(json.dumps(frames[-1],ensure_ascii=False,indent=2,allow_nan=False),encoding='utf-8')
(results/'demo_data.json').write_text(json.dumps(dataset,ensure_ascii=False,indent=2,allow_nan=False),encoding='utf-8')
(examples/(case['id']+'.json')).write_text(
json.dumps({k:v for k,v in data.items() if k!='frames'},
ensure_ascii=False,indent=2),encoding='utf-8')
(results/(case['id']+'_result.json')).write_text(
json.dumps(frames[-1],ensure_ascii=False,indent=2,allow_nan=False),encoding='utf-8')
(results/'demo_data.json').write_text(
json.dumps(dataset,ensure_ascii=False,indent=2,allow_nan=False),encoding='utf-8')
template=(ASSETS / 'q1.template.html').read_text(encoding='utf-8')
fragment=template.replace('/*__Q1_DATA__*/',json.dumps(dataset,ensure_ascii=False,allow_nan=False))
fragment=template.replace('/*__Q1_DATA__*/',
json.dumps(dataset,ensure_ascii=False,allow_nan=False))
(ROOT/'output'/'q1'/'question1_full.html').write_text(fragment,encoding='utf-8')
# A standalone companion uses the same fragment and computation results.
theme=(ASSETS / 'standalone.css').read_text(encoding='utf-8')
page='<!doctype html><html lang="zh-CN"><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>B题问题一交会定位</title><style>'+theme+'</style><body>'+fragment+'</body></html>'
page=('<!doctype html><html lang="zh-CN"><meta charset="utf-8">'
'<meta name="viewport" content="width=device-width,initial-scale=1">'
'<title>B题问题一交会定位</title>'
'<style>'+theme+'</style><body>'+fragment+'</body></html>')
(results/'question1.html').write_text(page,encoding='utf-8')
print(json.dumps({'cases':len(dataset),'fragment_bytes':len(fragment.encode('utf-8')),
'standalone':str(results/'question1.html')},ensure_ascii=False))
scripts/capture_practice.py
+11 -5
View File
@@ -43,8 +43,10 @@ class PracticeClient:
status = response.status
body = json.load(response)
# A retried action is recorded/consumed only once.
self.events.append(dict(path=path, request=payload, response=body, retries=attempt))
self.output.write_text(json.dumps(self.events,ensure_ascii=False,indent=2),encoding='utf-8')
self.events.append(dict(path=path, request=payload, response=body,
retries=attempt))
self.output.write_text(
json.dumps(self.events,ensure_ascii=False,indent=2),encoding='utf-8')
if status != 200 or body.get('accepted') is not True:
raise RuntimeError(f'动作未被接受: {status}, {body}')
if path == '/enter':
@@ -54,6 +56,7 @@ class PracticeClient:
if attempt == 2:
raise
time.sleep(.25)
raise URLError('网络重试次数已用尽')
def measure(self, p, channel):
return self.action('/measure',position={'x':p[0],'y':p[1]},channel=channel)
@@ -106,12 +109,15 @@ def main():
if len(observations)>=3:
break
data={'name':'官方问题3演练观测','source':'official_practice','case_code':args.case_code,
'channel':channel,'ground_truth':None,'measurements':[m.__dict__ for m in observations],
'channel':channel,'ground_truth':None,
'measurements':[m.__dict__ for m in observations],
'repeated_station_response':repeat,'result':solve(observations),
'scope':'仅采集问题一观测，不执行问题三全清除任务；没有调用clear'}
path=args.output/f'{args.case_code}_observations.json'
path.write_text(json.dumps(data,ensure_ascii=False,indent=2,allow_nan=False),encoding='utf-8')
print(json.dumps({'observations':str(path),'count':len(observations),'result':data['result']},ensure_ascii=False,indent=2))
path.write_text(json.dumps(data,ensure_ascii=False,indent=2,allow_nan=False),
encoding='utf-8')
print(json.dumps({'observations':str(path),'count':len(observations),
'result':data['result']},ensure_ascii=False,indent=2))
finally:
if entered:
client.action('/exit')
scripts/evaluate_rl_revision.py
+16 -8
View File
@@ -22,7 +22,8 @@ from referee import LocalSimulator, Source
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--split', choices=('development','validation','stress'), default='development')
parser.add_argument('--split', choices=('development','validation','stress'),
default='development')
parser.add_argument('--variants', default='baseline,legacy,revised')
parser.add_argument('--problems', default='3,4')
parser.add_argument('--model-dir', type=Path, default=Path('output/rl/review-20260911'))
@@ -49,7 +50,8 @@ def main():
repeated_measures=sum(r['repeated_measures'] for r in part))
result = dict(source='local_referee_only', config=asdict(config), split=args.split,
rows=rows, aggregates=aggregates)
(output/'results.json').write_text(json.dumps(result,ensure_ascii=False,indent=2),encoding='utf-8')
(output/'results.json').write_text(
json.dumps(result,ensure_ascii=False,indent=2),encoding='utf-8')
return aggregates
for problem in map(int, args.problems.split(',')):
scenes = []
@@ -71,7 +73,8 @@ def main():
scenes.append((f'{count}-{seed}',count,seed,sources))
for name,count,seed,sources in scenes:
scene_data = dict(sources=[asdict(s) for s in sources], error_seed=seed+2000000)
(output/f'q{problem}-{name}-scene.json').write_text(json.dumps(scene_data),encoding='utf-8')
scene_path = output/f'q{problem}-{name}-scene.json'
scene_path.write_text(json.dumps(scene_data),encoding='utf-8')
for variant in variants:
model = None
cls = BayesLoop if problem==3 else DirectionalExtension
@@ -81,13 +84,15 @@ def main():
model = (Path(f'output/rl/q{problem}_ppo.zip') if variant=='legacy'
else args.model_dir/f'q{problem}_masked.zip')
kwargs = dict(model_path=model, device='cpu')
sim = LocalSimulator(sources,robot_id='RL-AUDIT',spec=referee_spec(config),error_seed=seed+2000000)
sim = LocalSimulator(sources,robot_id='RL-AUDIT',spec=referee_spec(config),
error_seed=seed+2000000)
client = SimulatorClient('RL-AUDIT',config=config,transport=LocalSimTransport(sim))
start = perf_counter()
summary = cls(client,config,**kwargs).run()
events = client.log.action_events()
truth = {s.channel for s in sources}
successful = {e['request']['channel'] for e in events if e['response'].get('clear_result')=='success'}
successful = {e['request']['channel'] for e in events
if e['response'].get('clear_result')=='success'}
seen = set()
repeats = 0
for e in events:
@@ -102,12 +107,15 @@ def main():
and summary['exit_reason']=='user_exit' and not mismatches)
row = dict(case=name,problem=problem,variant=variant,valid=valid,
virtual_s=client.state.virtual_time_s,all_cleared=successful==truth,
measures=summary['measures'],failed_clears=summary['clears']-len(successful),
measures=summary['measures'],
failed_clears=summary['clears']-len(successful),
repeated_measures=repeats,wall_s=perf_counter()-start,
model_sha256=hashlib.sha256(model.read_bytes()).hexdigest() if model else None)
model_sha256=(hashlib.sha256(model.read_bytes()).hexdigest()
if model else None))
rows.append(row)
(output/f'q{problem}-{name}-{variant}.json').write_text(json.dumps(
dict(row=row,summary=summary,events=events,timing_mismatches=mismatches),ensure_ascii=False),encoding='utf-8')
dict(row=row,summary=summary,events=events,timing_mismatches=mismatches),
ensure_ascii=False),encoding='utf-8')
save()
print(json.dumps(row),flush=True)
print(json.dumps(save(),indent=2),flush=True)
scripts/pretrain_rl_expert.py
+3 -2
View File
@@ -88,7 +88,7 @@ def collect(problem: int, episodes: int, seed: int, config: ProblemConfig):
max_episode_steps=16)
observations, actions, groups = [], [], []
for episode in range(episodes):
obs, info = env.reset(seed=seed + episode)
obs, _info = env.reset(seed=seed + episode)
for _step in range(env.max_episode_steps + 2):
action = expert_action(env)
observations.append(obs)
@@ -108,7 +108,8 @@ def train_bc(x, y, *, epochs: int, batch_size: int, lr: float,
episodes = np.unique(groups)
if len(episodes) < 2:
raise ValueError('行为克隆至少需要两个独立episode才能划分验证集')
valid_groups = np.random.default_rng(0).permutation(episodes)[:max(1, int(len(episodes)*valid_frac))]
valid_groups = np.random.default_rng(0).permutation(episodes)
valid_groups = valid_groups[:max(1, int(len(episodes)*valid_frac))]
valid = torch.as_tensor(np.isin(groups, valid_groups), device=device)
x_t, y_t = torch.as_tensor(x, device=device), torch.as_tensor(y, device=device)
x_v, y_v = x_t[valid], y_t[valid]
scripts/render_q1_figures.py
+13 -9
View File
@@ -6,17 +6,17 @@ import math
from pathlib import Path
import re
import xml.etree.ElementTree as ET
from PIL import Image,ImageDraw,ImageFont
from PIL import Image,ImageDraw,ImageFont # pylint: disable=import-error # 可选依赖
ROOT=Path(__file__).resolve().parents[1]
S=2
FONT=ImageFont.truetype('C:/Windows/Fonts/msyh.ttc',12*S)
def render(path):
root=ET.fromstring(path.read_text(encoding='utf-8'))
def render(svg_path):
root=ET.fromstring(svg_path.read_text(encoding='utf-8'))
width,height=map(float,(root.get('width'),root.get('height')))
bg='#191c21' if 'dark' in path.stem else '#ffffff'
bg='#191c21' if 'dark' in svg_path.stem else '#ffffff'
image=Image.new('RGBA',(round(width*S),round(height*S)),bg)
clip=next(el for el in root.iter() if el.tag.endswith('clipPath'))[0]
clipbox=tuple(float(clip.get(k))*S for k in ('x','y','width','height'))
@@ -34,7 +34,8 @@ def render(path):
for child in el:draw(child,clipped or bool(el.get('clip-path')))
return
layer=Image.new('RGBA',image.size,(0,0,0,0));d=ImageDraw.Draw(layer)
num=lambda key,default=0:float(el.get(key,default))*S
def num(key,default=0):
return float(el.get(key,default))*S
fill=color(el.get('fill'),float(el.get('fill-opacity',1)))
stroke=color(el.get('stroke'),float(el.get('stroke-opacity',1)))
sw=max(1,round(num('stroke-width',1)))
@@ -43,13 +44,15 @@ def render(path):
dash=el.get('stroke-dasharray','none')
if dash=='none': d.line(xy,fill=stroke,width=sw)
else:
a,b=xy;dist=math.dist(a,b);unit=((b[0]-a[0])/max(dist,1e-15),(b[1]-a[1])/max(dist,1e-15))
a,b=xy;dist=math.dist(a,b)
unit=((b[0]-a[0])/max(dist,1e-15),(b[1]-a[1])/max(dist,1e-15))
on,off=[float(v)*S for v in dash.split()]
# Bound offscreen ray work to the finite exported viewport vicinity.
segments=min(100000,int(dist/(on+off))+1)
for i in range(segments):
lo=i*(on+off);hi=min(lo+on,dist)
d.line([(a[0]+lo*unit[0],a[1]+lo*unit[1]),(a[0]+hi*unit[0],a[1]+hi*unit[1])],fill=stroke,width=sw)
d.line([(a[0]+lo*unit[0],a[1]+lo*unit[1]),
(a[0]+hi*unit[0],a[1]+hi*unit[1])],fill=stroke,width=sw)
elif tag=='rect':
box=(num('x'),num('y'),num('x')+num('width'),num('y')+num('height'))
d.rectangle(box,fill=fill,outline=stroke,width=sw)
@@ -63,7 +66,8 @@ def render(path):
d.polygon(points,fill=fill)
if stroke:d.line(points+[points[0]],fill=stroke,width=sw,joint='curve')
elif tag=='path':
for x,y,axis,length in re.findall(r'M([\d.e+-]+),([\d.e+-]+)([hv])([\d.e+-]+)',el.get('d')):
pattern=r'M([\d.e+-]+),([\d.e+-]+)([hv])([\d.e+-]+)'
for x,y,axis,length in re.findall(pattern,el.get('d')):
x,y,length=float(x)*S,float(y)*S,float(length)*S
d.line([(x,y),(x+length,y) if axis=='h' else (x,y+length)],fill=stroke,width=sw)
elif tag=='text':
@@ -86,7 +90,7 @@ def render(path):
image=Image.alpha_composite(image,layer)
draw(root)
output=path.with_suffix('.png');image.convert('RGB').save(output)
output=svg_path.with_suffix('.png');image.convert('RGB').save(output)
return str(output.relative_to(ROOT))
scripts/run_q3_seven_practice.py
+3 -2
View File
@@ -32,8 +32,9 @@ def main():
metadata = {"source": "official_practice", "problem": 3,
"strategy": "seven_batch", "started_local": datetime.now().astimezone().isoformat(),
"config": asdict(config), "coverage_points": seven_points(),
"source_sha256": {str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest()
for p in paths}}
"source_sha256": {
str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest()
for p in paths}}
def save(name, value):
(args.output / name).write_text(json.dumps(value, ensure_ascii=False, indent=2,
allow_nan=False) + "\n", encoding="utf-8")
scripts/summarize_rl_revision.py
+12 -6
View File
@@ -12,7 +12,8 @@ from mcm26b.rl.features import build_observation
def main():
root = Path('output/rl/review-20260911')
records = json.loads((root/'evaluation/validation/final/results.json').read_text(encoding='utf-8'))
records = json.loads((root/'evaluation/validation/final/results.json')
.read_text(encoding='utf-8'))
rng = np.random.default_rng(20260911)
comparisons = {}
for problem in (3,4):
@@ -46,13 +47,16 @@ def main():
bc = {}
for problem in (3,4):
# 使用旧检查点和原始专家输入，比对原ReLU网络与拷贝后的Tanh网络。
checkpoint = torch.load(f'output/rl/bc/q{problem}_bc.pt',map_location='cpu',weights_only=True)
checkpoint = torch.load(f'output/rl/bc/q{problem}_bc.pt',map_location='cpu',
weights_only=True)
data = np.load(f'output/rl/bc/q{problem}_expert.npz',allow_pickle=False)
x = torch.from_numpy(data['obs'][::max(1,len(data['obs'])//1000)].copy())
def network(activation):
def network(activation, checkpoint=checkpoint):
net = torch.nn.Sequential(torch.nn.Linear(401,128),activation(),
torch.nn.Linear(128,128),activation(),torch.nn.Linear(128,60))
net.load_state_dict({k.removeprefix('net.'):v for k,v in checkpoint['state_dict'].items()})
state = {k.removeprefix('net.'):v for k,v in checkpoint['state_dict'].items()}
net.load_state_dict(state)
return net
with torch.no_grad():
before,after=network(torch.nn.ReLU)(x),network(torch.nn.Tanh)(x)
@@ -63,9 +67,11 @@ def main():
limitations='置信区间仅重采样固定模型的场景，不覆盖训练随机种子不确定性。',
code_sha256={str(p):hashlib.sha256(p.read_bytes()).hexdigest()
for p in sorted(Path('src/mcm26b/rl').glob('*.py'))})
(root/'review_evidence.json').write_text(json.dumps(report,ensure_ascii=False,indent=2),encoding='utf-8')
(root/'review_evidence.json').write_text(
json.dumps(report,ensure_ascii=False,indent=2),encoding='utf-8')
print(json.dumps(dict(comparisons=comparisons,feature_probe=features,bc_probe=bc,
official=[dict(case=r['case_code'],passed=r['passed']) for r in official]),indent=2))
official=[dict(case=r['case_code'],passed=r['passed'])
for r in official]),indent=2))
if __name__=='__main__':
scripts/train_hrl_high.py
+382 -107
View File
@@ -8,13 +8,31 @@
当前 HTTP 执行路径仍由高层宏动作终点决定，不把八方向推演当正式收益。
落盘保证（长训必需）：
- ``--checkpoint-every`` 每 N 个环境步在 ``<模型目录>/checkpoints/`` 落一个可续训
的 checkpoint（zip + 同名 json 元数据），只保留最近 ``--checkpoint-keep`` 个；
- 训练前端的「停止」是对**进程组**发 SIGTERM，因此 subproc worker 在 :func:`_make_env`
里显式忽略 SIGTERM/SIGINT，主进程收到信号后在 step 边界收尾并保存模型，
不会被 worker 先行退出打断；
- 任何异常（含被强杀之外的失败）都会先落一份 ``*_emergency.zip`` 再抛出，
不再出现"跑了很久什么都没有"。
- **急停可恢复**：``--recovery-every``（默认 2000 环境步）把一份滚动恢复点
``<模型目录>/checkpoints/<stem>_latest.zip`` 原子覆盖写；训练开始前先写一份
起步基线。因此即使进程被 SIGKILL / ``taskkill /F`` 强杀（没有任何 Python
收尾机会），也**最多只丢 ``--recovery-every`` 个环境步**，且磁盘上始终有一份
完整、可加载、可 ``--resume`` 的产物；
- **原子落盘**：模型 zip、旁挂 json 与 ``<run_dir>/resume.json`` 都是"临时文件 +
``os.replace``"，保存途中被杀不会留下半截文件，也不会破坏上一份完整快照；
- ``--checkpoint-every`` 每 N 个环境步在 ``<模型目录>/checkpoints/`` 另留一份历史
checkpoint（``<stem>_<步数>.zip``），只保留最近 ``--checkpoint-keep`` 个，供
返祖对比与多档恢复；
- ``<run_dir>/resume.json`` 是恢复的**权威指针**：无论上一刻是滚动恢复点、历史
checkpoint、优雅收尾还是异常兜底，它总指向一份可续训产物，并记录增量语义
（``timesteps`` 是追加步数）与生效过的 PPO 超参；
- ``--control-file`` 是训练前端的**跨平台**控制通道：前端原子写 JSON
（``paused`` / ``stop``），本脚本每个 step 轮询。暂停时训练在 step 边界挂起，
停止时先保存模型与元数据再退出；
- 信号（SIGTERM/SIGINT，Windows 上另有 SIGBREAK）仍可用：直接 kill 本脚本或
前端发进程组信号时同样走优雅收尾；subproc worker 在 :func:`_make_env` 里忽略
这些信号，保证它们活到主进程 ``vec_env.close()`` 之后再退出；
- 任何异常都会先落一份 ``*_emergency.zip`` + ``resume.json`` 再抛出，不再出现
"跑了很久什么都没有"。
为什么暂停/停止不靠信号：Windows 没有 SIGSTOP，且 ``os.kill(pid, SIGTERM)`` 会
退化成 ``TerminateProcess`` 硬杀——收尾分支根本不会执行。控制文件两端都只是
普通文件 IO，POSIX 与 Windows 行为一致（见 :mod:`mcm26b.process_control`）。
P1 smoke 示例：
python scripts/train_hrl_high.py \
@@ -29,6 +47,7 @@ from __future__ import annotations
import argparse
import hashlib
import json
import math
import os
import platform
import random
@@ -39,9 +58,10 @@ from pathlib import Path
from typing import Any
from mcm26b.config import ProblemConfig
from mcm26b.hrl.curriculum import STAGES, training_config
from mcm26b.hrl.curriculum import STAGES
from mcm26b.hrl.constants import (ACTION_LAYOUT_VERSION, FEATURE_VERSION,
HIGH_OBS_DIM, N_CANDIDATES, REWARD_VERSION)
from mcm26b.process_control import ControlState, read_control
def _stage_from_index(index: int):
@@ -68,24 +88,86 @@ def _reward_from_profile(profile: str):
raise ValueError(f"未知 reward-profile: {profile}")
def _ignore_stop_signals_in_worker() -> None:
"""subproc worker 忽略 SIGTERM/SIGINT，保证主进程能优雅收尾落盘。
def _stop_signals() -> list[int]:
"""本平台可用于"请求收尾"的信号集合。
训练前端的「停止」向整个进程组发 SIGTERM；若 worker 同时退出，主进程会在
保存模型之前就拿到断管/EOF 错误，整局训练白跑。worker 活在管道后面，主进程
收尾时通过 ``vec_env.close()`` 的 ``close`` 指令正常结束它们。
Windows 上没有 SIGTERM 的软语义（``os.kill`` 直接 TerminateProcess），但
``SIGBREAK`` 可以由 ``CTRL_BREAK_EVENT`` 触发并进入 Python 处理器；POSIX 上
SIGTERM/SIGINT 均可捕获。控制文件才是主通道，这里只是让"直接 kill 脚本"
也尽量走优雅收尾。
"""
signals = [signal.SIGTERM, signal.SIGINT]
sigbreak = getattr(signal, "SIGBREAK", None)
if sigbreak is not None:
signals.append(sigbreak)
return signals
def _ignore_stop_signals_in_worker() -> None:
"""subproc worker 忽略收尾信号，保证主进程能优雅落盘。
无论信号来自进程组 SIGTERM 还是 Windows ``CTRL_BREAK_EVENT``，若 worker
同时退出，主进程会在保存模型之前就拿到断管/EOF 错误，整局训练白跑。worker
活在管道后面，主进程收尾时通过 ``vec_env.close()`` 的 ``close`` 指令正常
结束它们。
"""
try:
import multiprocessing
if multiprocessing.current_process().name == "MainProcess":
return
for signum in (signal.SIGTERM, signal.SIGINT):
for signum in _stop_signals():
signal.signal(signum, signal.SIG_IGN)
except (ImportError, ValueError, OSError): # 非主线程/平台不支持：不影响训练
return
def _make_env(problem: int, stage_index: int, seed: int,
class _ControlPoll:
"""控制文件轮询状态机（纯标准库，可脱离 SB3 单测）。
``poll()`` 每 ``poll_every`` 次调用才真正读盘，避免每个 env step 都做一次
文件 IO；``wait_while_paused()`` 阻塞到暂停解除或收到 stop，并累计暂停秒数
（``--max-wall-s`` 要把这段扣掉，否则"挂起一小时回来就判超时"）。
"""
def __init__(self, path: Path, *, poll_every: int = 32,
poll_s: float = 0.2):
self.path = Path(path)
self.poll_every = max(1, int(poll_every))
self.poll_s = max(0.02, float(poll_s))
self.paused_total_s = 0.0
self._calls = 0
self._state = ControlState()
def poll(self) -> ControlState:
"""按抽样间隔返回最近一次控制状态。"""
self._calls += 1
if self._calls % self.poll_every:
return self._state
self._state = read_control(self.path)
return self._state
def wait_while_paused(self, on_pause, on_resume) -> bool:
"""阻塞到继续或停止；返回 True 表示期间收到了 stop。
``on_pause`` / ``on_resume`` 用来向前端回显事件（训练脚本的 JSON 行）。
"""
on_pause()
started = time.monotonic()
stopped = False
while True:
time.sleep(self.poll_s)
self._state = read_control(self.path)
if self._state.stop:
stopped = True
break
if not self._state.paused:
break
self.paused_total_s += time.monotonic() - started
on_resume()
return stopped
def _make_env(problem: int, stage_index: int,
monitor_path: str | None, profile: str,
reward_profile: str):
"""顶层工厂：供 DummyVecEnv / SubprocVecEnv 调用（必须是可 pickle 的）。"""
@@ -112,7 +194,7 @@ def _build_vec_env(args, monitor_paths: list[str]):
from functools import partial
factories = [
partial(_make_env, args.problem, args.stage, args.seed + rank,
partial(_make_env, args.problem, args.stage,
monitor_paths[rank], args.config_profile, args.reward_profile)
for rank in range(args.envs)
]
@@ -161,6 +243,52 @@ def _policy_hyperparams(model) -> dict[str, Any]:
}
def _progress_payload(model, start: int) -> dict[str, Any]:
"""组装 ``--progress-json`` 的单行负载。
纯函数（只读 model 的 logger / ep_info_buffer），便于脱离训练循环测试；
NaN / ±inf 一律折算成 None，保证 ``json.dumps(allow_nan=False)`` 不会炸。
"""
values = getattr(model.logger, "name_to_value", {}) or {}
def _num(key, default=None):
value = values.get(key)
if value is None:
return default
try:
value = float(value)
except (TypeError, ValueError):
return default
if not math.isfinite(value):
return default
return value
ep_rewards: list[float] = []
ep_lengths: list[float] = []
for item in getattr(model, "ep_info_buffer", []) or []:
try:
ep_rewards.append(float(item.get("r")))
ep_lengths.append(float(item.get("l")))
except (TypeError, ValueError, AttributeError):
continue
return {
"event": "progress",
"timesteps": int(model.num_timesteps),
"start_timesteps": int(start),
"n_updates": int(getattr(model, "_n_updates", 0) or 0),
"ep_rew_mean": (sum(ep_rewards) / len(ep_rewards)
if ep_rewards else None),
"ep_len_mean": (sum(ep_lengths) / len(ep_lengths)
if ep_lengths else None),
"value_loss": _num("train/value_loss"),
"policy_gradient_loss": _num("train/policy_gradient_loss"),
"approx_kl": _num("train/approx_kl"),
"entropy_loss": _num("train/entropy_loss"),
"explained_variance": _num("train/explained_variance"),
"fps": _num("time/fps"),
}
def _model_metadata(*, args, wall_s: float, n_envs: int,
actual_timesteps: int) -> dict[str, Any]:
import numpy
@@ -190,12 +318,12 @@ def _model_metadata(*, args, wall_s: float, n_envs: int,
"config_profile": args.config_profile,
"fidelity": _profile_fidelity(args.config_profile),
"reward_profile": args.reward_profile,
"reward_version": REWARD_VERSION,
"local_selector": args.local_selector,
"local_model": args.local_model,
"checkpoint_every": int(getattr(args, "checkpoint_every", 0) or 0),
"checkpoint_keep": int(getattr(args, "checkpoint_keep", 0) or 0),
"checkpoint_dir": getattr(args, "checkpoint_dir", None),
"recovery_every": int(getattr(args, "recovery_every", 0) or 0),
"device": args.device,
"wall_seconds": round(wall_s, 3),
"max_wall_s": float(args.max_wall_s),
@@ -211,6 +339,68 @@ def _model_metadata(*, args, wall_s: float, n_envs: int,
}
def _write_json_atomic(path: Path, payload: dict[str, Any]) -> None:
"""原子写 JSON：同目录 ``.tmp-`` 前缀临时文件 + ``os.replace``。
急停（SIGKILL / ``taskkill /F``）会绕过所有 Python 清理逻辑，保存途中被杀
是完全可能的。原子替换保证**真实文件名下要么是旧的完整文件、要么是新的
完整文件**，不会出现半截 JSON 让续训读到一半就崩。
"""
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_name(f".tmp-{path.name}")
tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2),
encoding="utf-8")
os.replace(tmp, path)
def _save_model_atomic(model, target: Path) -> Path:
"""把 SB3 模型原子地写到 ``target``，返回实际落盘路径。
先写 ``.tmp-<name>``（同样以 ``.zip`` 结尾，SB3 不会再加后缀），完整写完后
``os.replace`` 覆盖目标。于是：
- 保存途中被强杀 → 目标路径仍是上一份**完整**的快照，可继续复训；
- 保存成功 → 目标路径立即变成新快照，不存在"看起来存在但打不开"的窗口。
临时名带 ``.tmp-`` 前缀，因此不会被 ``*_ckpt_*.zip`` 的清理逻辑误当成正式
checkpoint。
"""
target = Path(target)
target.parent.mkdir(parents=True, exist_ok=True)
tmp = target.with_name(f".tmp-{target.name}")
model.save(str(tmp))
if not tmp.exists(): # SB3 对未知后缀可能自行补 .zip
candidate = tmp.with_name(tmp.name + ".zip")
if candidate.exists():
tmp = candidate
os.replace(tmp, target)
return target
def _write_resume_hint(*, run_dir: Path, model_path: Path, timesteps: int,
reason: str, model=None) -> dict[str, Any]:
"""把"从哪继续"写进 ``<run_dir>/resume.json``。
这是急停恢复的**权威指针**：无论上一刻是周期 checkpoint、滚动恢复点、
优雅收尾还是异常兜底，这个文件总指向一份完整、可 ``--resume`` 的产物，
并带上续训语义（``timesteps`` 是增量步数）与生效过的 PPO 超参。
"""
hint: dict[str, Any] = {
"event": "resume_hint",
"model": str(model_path),
"timesteps": int(timesteps),
"timesteps_semantics": "additional",
"reason": str(reason),
"created_unix_s": time.time(),
"resume_args": ["--resume", "--model", str(model_path)],
}
if model is not None:
hint["ppo_hyperparams"] = _policy_hyperparams(model)
_write_json_atomic(run_dir / "resume.json", hint)
return hint
def _write_metadata(*, model_path: Path, args, wall_s: float, n_envs: int,
actual_timesteps: int, monitor_paths: list[str],
model=None,
@@ -227,14 +417,18 @@ def _write_metadata(*, model_path: Path, args, wall_s: float, n_envs: int,
"reward_profile=prompt 使用原始 §5.2 塑形。模型元数据已固化口径。")
if extra:
metadata.update(extra)
model_path.with_suffix(".json").write_text(
json.dumps(metadata, ensure_ascii=False, indent=2), encoding="utf-8")
_write_json_atomic(model_path.with_suffix(".json"), metadata)
return metadata
def _prune_checkpoints(directory: Path, stem: str, keep: int) -> list[str]:
"""只保留最近 ``keep`` 个 checkpoint（zip+json 同删），返回被删的 zip 路径。"""
files = sorted(directory.glob(f"{stem}_*.zip"),
"""只保留最近 ``keep`` 个 checkpoint（zip+json 同删），返回被删的 zip 路径。
只匹配 ``<stem>_<步数>.zip``；滚动恢复点 ``<stem>_latest.zip`` 与临时文件
``.tmp-*.zip`` 都不匹配，因此永远不会被历史清理误删。
"""
files = sorted((path for path in directory.glob(f"{stem}_[0-9]*.zip")
if path.stem.rsplit("_", 1)[-1].isdigit()),
key=lambda path: path.stat().st_mtime, reverse=True)
removed: list[str] = []
for stale in files[max(1, int(keep)):]:
@@ -267,7 +461,7 @@ def _evaluate_env(model, *, problem: int, stage_index: int,
while not done:
action, _state = model.predict(
obs, action_masks=env.action_masks(), deterministic=True)
obs, reward, terminated, truncated, info = env.step(int(action))
obs, _, terminated, truncated, info = env.step(int(action))
done = bool(terminated or truncated)
steps += 1
# 课程阶段的小场景只需证明全部频道 accounted；完整阶段至少应清掉
@@ -299,7 +493,7 @@ def _evaluate_env(model, *, problem: int, stage_index: int,
def _evaluate_manifest(model_path: Path, manifest_path: str, *,
limit: int, problem: int) -> dict[str, Any]:
limit: int) -> dict[str, Any]:
"""可选：用 benchmark_hrl 的同一套真值隔离评测在固定 manifest 上跑已训模型。"""
scripts_dir = Path(__file__).resolve().parent
if str(scripts_dir) not in sys.path:
@@ -404,26 +598,72 @@ def run_training(args) -> int:
"命令行 rollout-steps/batch-size/n-epochs 被忽略")},
ensure_ascii=False), file=sys.stderr, flush=True)
# ------------------------------------------------------------ 优雅停止
# 训练前端「停止」= killpg(SIGTERM)：worker 已在 _make_env 里忽略该信号，
# 主进程在此装处理器，等当前 step 结束后收尾落盘。
# ------------------------------------------------------ 优雅暂停 / 停止
# 主通道是控制文件（跨平台）；信号是兜底：直接 kill 本进程时也走同一段收尾。
stop_state: dict[str, Any] = {"requested": False, "signal": None}
previous_handlers: dict[int, Any] = {}
def _on_stop_signal(signum, _frame):
def _request_stop(source: str) -> None:
if stop_state["requested"]:
return
stop_state["requested"] = True
stop_state["signal"] = signal.Signals(signum).name
print(json.dumps({"event": "stop_requested",
"signal": signal.Signals(signum).name}), flush=True)
stop_state["signal"] = source
print(json.dumps({"event": "stop_requested", "signal": source}), flush=True)
for _signum in (signal.SIGTERM, signal.SIGINT):
def _on_stop_signal(signum, _frame):
_request_stop(signal.Signals(signum).name)
for _signum in _stop_signals():
try:
previous_handlers[_signum] = signal.signal(_signum, _on_stop_signal)
except (ValueError, OSError): # 非主线程/平台不支持：不阻塞训练
pass
class _ControlChannel(BaseCallback):
"""把 :class:`_ControlPoll` 接到 SB3 的 step 回调上。
为什么不用信号：Windows 没有 SIGSTOP，且 ``os.kill(SIGTERM)`` 会退化成
``TerminateProcess`` 硬杀，收尾分支根本不会执行。控制文件在 POSIX 与
Windows 上都只是普通文件 IO，语义完全一致。
暂停是**协作式**的：在 ``_on_step`` 内阻塞轮询，训练因此停在 step 边界，
进程仍存活、显存仍占用，但不消耗算力；收到 stop 立刻转入收尾落盘。
"""
def __init__(self, path: Path, *, poll_every: int = 32,
poll_s: float = 0.2):
super().__init__()
self.poller = _ControlPoll(path, poll_every=poll_every,
poll_s=poll_s)
def paused_seconds(self) -> float:
return self.poller.paused_total_s
def _event(self, name: str) -> None:
print(json.dumps({"event": name,
"timesteps": int(self.model.num_timesteps)}),
flush=True)
def _on_step(self) -> bool:
state = self.poller.poll()
if state.stop:
_request_stop("control_file")
return False
if state.paused:
stopped = self.poller.wait_while_paused(
lambda: self._event("paused"),
lambda: self._event("resumed"))
if stopped:
_request_stop("control_file")
return False
return not stop_state["requested"]
return True
control_path = getattr(args, "control_file", None)
control_cb = _ControlChannel(Path(control_path)) if control_path else None
class _GracefulStop(BaseCallback):
"""收到停止信号后在最近一次 step 边界结束 rollout，让模型能落盘。"""
"""收到停止请求后在最近一次 step 边界结束 rollout，让模型能落盘。"""
def _on_step(self) -> bool:
if stop_state["requested"]:
@@ -435,15 +675,22 @@ def run_training(args) -> int:
return True
class _WallClockStop(BaseCallback):
def __init__(self, max_wall_s: float):
"""墙钟硬上限；暂停时间不计入，避免"挂起一小时回来就判超时"。"""
def __init__(self, max_wall_s: float, paused_clock=None):
super().__init__()
self.max_wall_s = max_wall_s
self.start = time.monotonic()
self.paused_clock = paused_clock
def paused_seconds(self) -> float:
return float(self.paused_clock()) if self.paused_clock else 0.0
def _on_step(self) -> bool:
if time.monotonic() - self.start >= self.max_wall_s:
elapsed = time.monotonic() - self.start - self.paused_seconds()
if elapsed >= self.max_wall_s:
print(json.dumps({"event": "wall_clock_stop",
"elapsed_s": time.monotonic() - self.start}))
"elapsed_s": elapsed}))
return False
return True
@@ -473,44 +720,7 @@ def run_training(args) -> int:
return True
self.last_emit = current
self.emitted += 1
values = getattr(self.model.logger, "name_to_value", {}) or {}
def _num(key, default=None):
value = values.get(key)
if value is None:
return default
try:
value = float(value)
except (TypeError, ValueError):
return default
if value != value or value in (float("inf"), float("-inf")):
return default
return value
ep_rewards = []
ep_lengths = []
for item in getattr(self.model, "ep_info_buffer", []) or []:
try:
ep_rewards.append(float(item.get("r")))
ep_lengths.append(float(item.get("l")))
except (TypeError, ValueError, AttributeError):
continue
payload = {
"event": "progress",
"timesteps": current,
"start_timesteps": self.start,
"n_updates": int(getattr(self.model, "_n_updates", 0) or 0),
"ep_rew_mean": (sum(ep_rewards) / len(ep_rewards)
if ep_rewards else None),
"ep_len_mean": (sum(ep_lengths) / len(ep_lengths)
if ep_lengths else None),
"value_loss": _num("train/value_loss"),
"policy_gradient_loss": _num("train/policy_gradient_loss"),
"approx_kl": _num("train/approx_kl"),
"entropy_loss": _num("train/entropy_loss"),
"explained_variance": _num("train/explained_variance"),
"fps": _num("time/fps"),
}
payload = _progress_payload(self.model, self.start)
print(json.dumps(payload, ensure_ascii=False, allow_nan=False),
flush=True)
return True
@@ -521,38 +731,79 @@ def run_training(args) -> int:
checkpoint_stem = f"{model_path.stem}_ckpt"
class _CheckpointJSON(BaseCallback):
"""每 N 个环境步落一个可续训 checkpoint（zip + 同名元数据）。
"""可续训落盘：滚动恢复点 + 周期 checkpoint + resume.json 指针。
只保留最近 ``keep`` 个；checkpoint 与最终模型共用同名 json 契约，
因此 ``--resume <checkpoint.zip>`` 直接可用。
- **滚动恢复点** ``<stem>_latest.zip``：每 ``recovery_every`` 个环境步
原子覆盖写一次，训练开始前也先写一份。它保证"急停"（SIGKILL /
``taskkill /F``，没有任何 Python 清理机会）之后**最多只丢
``recovery_every`` 个环境步**，而且文件始终完整可加载。
- **周期 checkpoint** ``<stem>_<步数>.zip``：每 ``every_steps`` 步留一份，
只保留最近 ``keep`` 个，用于返祖对比与多档恢复。
- 每次落盘都刷新 ``<run_dir>/resume.json``：无论怎么停，都有一份权威的
"从哪继续"指针（含增量语义与生效过的 PPO 超参）。
"""
def __init__(self, every_steps: int, keep: int):
def __init__(self, every_steps: int, keep: int, recovery_every: int):
super().__init__()
self.every_steps = max(1, int(every_steps))
self.every_steps = max(0, int(every_steps))
self.keep = max(1, int(keep))
self.recovery_every = max(0, int(recovery_every))
self.last_saved = int(model.num_timesteps)
self.last_recovery = int(model.num_timesteps)
self.saved: list[str] = []
self.latest_path = checkpoints_dir / f"{checkpoint_stem}_latest.zip"
def write_recovery(self, reason: str) -> None:
"""立刻写一份滚动恢复点；训练开始前调用即"起步基线"。
用闭包里的 ``model`` 而不是 ``self.model``：``BaseCallback.model``
要等 ``learn()`` 时才被赋值，而起步基线恰恰在 ``learn()`` 之前写。
"""
current = int(model.num_timesteps)
self.last_recovery = current
self._write_recovery(current, reason)
def _on_step(self) -> bool:
current = int(self.model.num_timesteps)
if current - self.last_saved < self.every_steps:
return True
self.last_saved = current
self._write(current)
current = int(model.num_timesteps)
if (self.recovery_every > 0
and current - self.last_recovery >= self.recovery_every):
self.last_recovery = current
self._write_recovery(current, "interval")
if (self.every_steps > 0
and current - self.last_saved >= self.every_steps):
self.last_saved = current
self._write_interval(current)
return True
def _write(self, current: int) -> None:
checkpoints_dir.mkdir(parents=True, exist_ok=True)
target = checkpoints_dir / f"{checkpoint_stem}_{current:09d}.zip"
self.model.save(str(target))
def _persist(self, target: Path, current: int, reason: str,
extra: dict[str, Any]) -> None:
"""原子落盘 zip + 元数据，并刷新 resume.json 指针。"""
_save_model_atomic(model, target)
_write_metadata(
model_path=target, args=args,
wall_s=time.monotonic() - started, n_envs=args.envs,
actual_timesteps=current, monitor_paths=monitor_paths,
model=self.model,
extra={"checkpoint": True, "checkpoint_timesteps": current,
"checkpoint_of": str(model_path)})
model=model,
extra={"checkpoint_timesteps": current,
"checkpoint_reason": reason,
"checkpoint_of": str(model_path), **extra})
_write_resume_hint(run_dir=run_dir, model_path=target,
timesteps=current, reason=reason,
model=model)
def _write_recovery(self, current: int, reason: str) -> None:
checkpoints_dir.mkdir(parents=True, exist_ok=True)
self._persist(self.latest_path, current, reason,
{"recovery": True})
print(json.dumps({"event": "recovery_checkpoint",
"reason": reason, "timesteps": current,
"model": str(self.latest_path)},
ensure_ascii=False), flush=True)
def _write_interval(self, current: int) -> None:
checkpoints_dir.mkdir(parents=True, exist_ok=True)
target = checkpoints_dir / f"{checkpoint_stem}_{current:09d}.zip"
self._persist(target, current, "interval", {"checkpoint": True})
removed = _prune_checkpoints(checkpoints_dir, checkpoint_stem,
self.keep)
self.saved.append(str(target))
@@ -562,15 +813,24 @@ def run_training(args) -> int:
progress_every = (args.progress_every if args.progress_every > 0
else args.rollout_steps * args.envs)
checkpoint_cb = None
if args.checkpoint_every > 0:
checkpoint_cb = _CheckpointJSON(args.checkpoint_every,
args.checkpoint_keep)
callbacks = [_GracefulStop(), _WallClockStop(args.max_wall_s)]
if checkpoint_cb is not None:
callbacks.append(checkpoint_cb)
checkpoint_cb = _CheckpointJSON(args.checkpoint_every, args.checkpoint_keep,
args.recovery_every)
if args.recovery_every > 0:
# 起步基线：训练还没跑就急停，也至少有一份可加载、可续训的产物
checkpoint_cb.write_recovery("baseline")
# 顺序有语义：控制通道最先（暂停/停止要在本步立刻生效），进度次之，
# 墙钟上限要能读到控制通道累计的暂停秒数，checkpoint 放最后。
callbacks: list[Any] = []
if control_cb is not None:
callbacks.append(control_cb)
if args.progress_json:
callbacks.insert(0, _ProgressJSON(progress_every))
callbacks.append(_ProgressJSON(progress_every))
callbacks.append(_GracefulStop())
callbacks.append(_WallClockStop(
args.max_wall_s,
paused_clock=(control_cb.paused_seconds if control_cb is not None
else None)))
callbacks.append(checkpoint_cb)
actual_timesteps = int(model.num_timesteps)
try:
@@ -587,7 +847,7 @@ def run_training(args) -> int:
}
emergency = run_dir / f"{model_path.stem}_emergency.zip"
try:
model.save(str(emergency))
_save_model_atomic(model, emergency)
_write_metadata(model_path=emergency, args=args,
wall_s=time.monotonic() - started,
n_envs=args.envs,
@@ -595,27 +855,34 @@ def run_training(args) -> int:
monitor_paths=monitor_paths, model=model,
extra={"emergency": True,
"error": failure["error"]})
_write_resume_hint(run_dir=run_dir, model_path=emergency,
timesteps=int(model.num_timesteps),
reason="emergency", model=model)
failure["emergency_model"] = str(emergency)
except Exception as save_exc: # 落盘失败不能再掩盖原始异常
failure["emergency_save_error"] = (
f"{type(save_exc).__name__}: {save_exc}")
(run_dir / "training_failure.json").write_text(
json.dumps(failure, ensure_ascii=False, indent=2), encoding="utf-8")
_write_json_atomic(run_dir / "training_failure.json", failure)
print(json.dumps(failure, ensure_ascii=False), file=sys.stderr,
flush=True)
raise
wall_s = time.monotonic() - started
actual_timesteps = int(model.num_timesteps)
stopped_by_signal = bool(stop_state["requested"])
model.save(str(model_path))
_save_model_atomic(model, model_path)
metadata = _write_metadata(
model_path=model_path, args=args, wall_s=wall_s, n_envs=args.envs,
actual_timesteps=actual_timesteps, monitor_paths=monitor_paths,
model=model,
extra={"stopped_by_signal": stopped_by_signal,
"stop_signal": stop_state["signal"],
"checkpoint_files": (checkpoint_cb.saved
if checkpoint_cb is not None else [])})
"checkpoint_files": checkpoint_cb.saved,
"recovery_model": str(checkpoint_cb.latest_path)})
# 最终模型也刷新一次 resume.json：优雅收尾后"从哪继续"指向最终模型
_write_resume_hint(run_dir=run_dir, model_path=model_path,
timesteps=actual_timesteps,
reason=("stopped" if stopped_by_signal else "finished"),
model=model)
metadata_path = model_path.with_suffix(".json")
vec_env.close()
# 收尾阶段恢复默认处理器：再次 Ctrl+C / 停止可立即强杀，不再等待评估。
@@ -637,8 +904,7 @@ def run_training(args) -> int:
if args.eval_manifest:
try:
manifest_eval = _evaluate_manifest(
model_path, args.eval_manifest, limit=args.eval_manifest_limit,
problem=args.problem)
model_path, args.eval_manifest, limit=args.eval_manifest_limit)
except Exception as exc:
manifest_eval = {"error": f"{type(exc).__name__}: {exc}"}
metadata["evaluation"] = evaluation
@@ -685,10 +951,17 @@ def main(argv=None) -> int:
help="time=P0.4 纯时间口径；prompt=原始 §5.2 塑形对照")
parser.add_argument("--progress-json", action="store_true",
help="向训练前端输出单行 JSON 进度（UI 自动启用）")
parser.add_argument("--control-file", default=None,
help="训练前端控制文件（JSON：paused/stop）；"
"跨平台暂停与优雅停止的主通道")
parser.add_argument("--progress-every", type=int, default=0,
help="JSON 进度间隔环境步数；0=每个 rollout")
parser.add_argument("--checkpoint-every", type=int, default=20000,
help="每 N 个环境步落一个 checkpoint；0=关闭（默认 20000）")
help="每 N 个环境步留一个历史 checkpoint；0=关闭（默认 20000）")
parser.add_argument("--recovery-every", type=int, default=2000,
help="每 N 个环境步原子覆盖写一份滚动恢复点 "
"<模型目录>/checkpoints/<stem>_latest.zip；"
"急停后最多丢 N 步。0=关闭（默认 2000）")
parser.add_argument("--checkpoint-keep", type=int, default=3,
help="checkpoint 保留最近 K 个（默认 3）")
parser.add_argument("--checkpoint-dir", default=None,
@@ -703,6 +976,8 @@ def main(argv=None) -> int:
parser.error("eval-episodes 不能为负")
if args.checkpoint_every < 0:
parser.error("checkpoint-every 不能为负")
if args.recovery_every < 0:
parser.error("recovery-every 不能为负")
if args.checkpoint_keep < 1:
parser.error("checkpoint-keep 至少为 1")
return run_training(args)
scripts/train_hrl_low.py
-1
View File
@@ -82,7 +82,6 @@ def bc_validation_mask(episode_ids, *, seed: int):
def train_bc(x, y, episode_ids, *, epochs: int, batch_size: int, lr: float,
device: str, seed: int):
import numpy as np
import torch
BCMlp = _bc_module(LOW_OBS_DIM, LOW_NUM_ACTIONS, HIDDEN)
model = BCMlp().to(device)
scripts/train_rl.py
+2 -1
View File
@@ -130,7 +130,8 @@ def main(argv=None) -> int:
for rank in range(max(1, args.envs))])
if args.init:
metadata = json.loads(Path(args.init).with_suffix('.json').read_text(encoding='utf-8'))
if metadata.get('algorithm', 'ppo') != args.algorithm or metadata.get('feature_version') != 2:
if (metadata.get('algorithm', 'ppo') != args.algorithm
or metadata.get('feature_version') != 2):
parser.error('初始化模型的算法/特征版本与当前训练不符，请重训或提供v2初始化模型')
model = model_cls.load(args.init, env=vec_env, device=device)
else:
scripts/tune_q3_bayes.py
+1 -1
View File
@@ -44,7 +44,7 @@ def evaluate(bayes_params: dict) -> tuple[float, dict]:
config = replace(BASE, bayes=replace(BASE.bayes, **bayes_params))
times = []
failures = 0
for name, sources, error_seed in SCENES:
for _name, sources, error_seed in SCENES:
sim = LocalSimulator(sources, robot_id="TUNE", config=config,
error_seed=error_seed)
client = SimulatorClient("TUNE", config=config,
scripts/verify_q3_practice.py
+49 -36
View File
@@ -33,7 +33,10 @@ class StrictPlayback(PlaybackTransport):
raise AssertionError("Replay attempted an extra action")
expected = self.events[self.index]
actual_request = json.loads(body)
without_id = lambda value: {k: v for k, v in value.items() if k != "request_id"}
def without_id(value):
return {k: v for k, v in value.items() if k != "request_id"}
if (urlsplit(url).path != expected["path"] or
without_id(actual_request) != without_id(expected["request"])):
raise AssertionError(f"Replay action differs at index {self.index}")
@@ -66,6 +69,47 @@ def save_json(path, value):
allow_nan=False) + "\n", encoding="utf-8")
def timing_rows(events):
"""按官方附件 2 第 4.1 节逐动作累计虚拟时间（微秒），返回明细与汇总。"""
position, channel, virtual_us = (0.0, 0.0), 1, 0
distance_total, switch_total, move_total_us = 0.0, 0, 0
rows = []
previous_recorded = 0.0
max_local_gap = 0.0
for i, event in enumerate(events):
request, response = event["request"], event["response"]
path = event["path"]
distance, switches, action_s = 0.0, 0, 0
if path in ("/measure", "/clear"):
target = (request["position"]["x"], request["position"]["y"])
distance = math.dist(position, target)
position = target
if path == "/measure":
switches = int(channel != request["channel"])
channel = request["channel"]
action_s = 5
else:
action_s = 5 if response["clear_result"] == "success" else 3
move_us = round(distance / 5 * 1_000_000)
virtual_us += move_us + (switches + action_s) * 1_000_000
distance_total += distance
switch_total += switches
move_total_us += move_us
recorded = response["virtual_time_s"]
gap_us = round(recorded * 1_000_000) - virtual_us
max_local_gap = max(max_local_gap,
abs(recorded - previous_recorded - distance/5 - switches - action_s))
previous_recorded = recorded
rows.append({"index": i, "path": path, "request_id": request["request_id"],
"channel": request.get("channel", ""), "x": position[0], "y": position[1],
"result": response.get("measure_result", response.get("clear_result", "")),
"svd_deg": response.get("svd_deg", ""), "distance_m": distance,
"move_us": move_us, "switch_s": switches, "action_s": action_s,
"recomputed_us": virtual_us, "recorded_s": recorded, "gap_us": gap_us})
return rows, virtual_us, distance_total, move_total_us, switch_total, previous_recorded, \
max_local_gap
def audit(folder: Path) -> dict:
records = [json.loads(line) for line in
(folder / "session.jsonl").read_text(encoding="utf-8").splitlines()]
@@ -130,40 +174,8 @@ def audit(folder: Path) -> dict:
# Official attachment 2 section 4.1: accumulate virtual time in microseconds.
# Round each move to the nearest microsecond, then add integer action costs.
position, channel, virtual_us = (0.0, 0.0), 1, 0
distance_total, switch_total, move_total_us = 0.0, 0, 0
rows = []
previous_recorded = 0.0
max_local_gap = 0.0
for i, event in enumerate(events):
request, response, path = event["request"], event["response"], event["path"]
distance, switches, action_s = 0.0, 0, 0
if path in ("/measure", "/clear"):
target = (request["position"]["x"], request["position"]["y"])
distance = math.dist(position, target)
position = target
if path == "/measure":
switches = int(channel != request["channel"])
channel = request["channel"]
action_s = 5
else:
action_s = 5 if response["clear_result"] == "success" else 3
move_us = round(distance / 5 * 1_000_000)
virtual_us += move_us + (switches + action_s) * 1_000_000
distance_total += distance
switch_total += switches
move_total_us += move_us
recorded = response["virtual_time_s"]
gap_us = round(recorded * 1_000_000) - virtual_us
max_local_gap = max(max_local_gap,
abs(recorded - previous_recorded - distance/5 - switches - action_s))
previous_recorded = recorded
rows.append({"index": i, "path": path, "request_id": request["request_id"],
"channel": request.get("channel", ""), "x": position[0], "y": position[1],
"result": response.get("measure_result", response.get("clear_result", "")),
"svd_deg": response.get("svd_deg", ""), "distance_m": distance,
"move_us": move_us, "switch_s": switches, "action_s": action_s,
"recomputed_us": virtual_us, "recorded_s": recorded, "gap_us": gap_us})
rows, virtual_us, distance_total, move_total_us, switch_total, previous_recorded, \
max_local_gap = timing_rows(events)
checks["all_timestamps_match_microsecond_recomputation"] = all(r["gap_us"] == 0 for r in rows)
real_s = (events[-1]["response"]["real_timestamp_ms"] -
events[0]["response"]["real_timestamp_ms"]) / 1000
@@ -206,7 +218,8 @@ def audit(folder: Path) -> dict:
"clear_time_s": sum(r["action_s"] for r in rows if r["path"] == "/clear"),
"timing_audit": {
"original_unquantized_mismatch_count": len(raw_timing["mismatches"]),
"original_unquantized_final_gap_s": previous_recorded - raw_timing["final_virtual_time_s"],
"original_unquantized_final_gap_s":
previous_recorded - raw_timing["final_virtual_time_s"],
"max_single_action_unquantized_gap_s": max_local_gap,
"microsecond_mismatch_count": sum(r["gap_us"] != 0 for r in rows),
"explanation": "原复算累计未量化浮点耗时；按动作移动耗时取最近整数微秒后，所有官方时刻完全一致。",
scripts/verify_rl_practice.py
+4 -4
View File
@@ -116,8 +116,8 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("folder", type=Path)
args = parser.parse_args()
report = audit(args.folder)
print(json.dumps({"passed": report["passed"], "case_code": report["case_code"],
"problem": report["problem"], "checks": report["checks"]},
result = audit(args.folder)
print(json.dumps({"passed": result["passed"], "case_code": result["case_code"],
"problem": result["problem"], "checks": result["checks"]},
ensure_ascii=False, indent=2))
raise SystemExit(0 if report["passed"] else 1)
raise SystemExit(0 if result["passed"] else 1)