sloray/mcm-26-b
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

holy fuking shit

Browse Source
This commit is contained in:
Horldsence
2026-09-12 01:56:34 +08:00
parent 36f354736c
commit d975309a81
83 changed files with 2748 additions and 896 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/evaluate_rl_revision.py
+16 -8
View File
@@ -22,7 +22,8 @@ from referee import LocalSimulator, Source
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--split', choices=('development','validation','stress'), default='development')
parser.add_argument('--split', choices=('development','validation','stress'),
default='development')
parser.add_argument('--variants', default='baseline,legacy,revised')
parser.add_argument('--problems', default='3,4')
parser.add_argument('--model-dir', type=Path, default=Path('output/rl/review-20260911'))
@@ -49,7 +50,8 @@ def main():
repeated_measures=sum(r['repeated_measures'] for r in part))
result = dict(source='local_referee_only', config=asdict(config), split=args.split,
rows=rows, aggregates=aggregates)
(output/'results.json').write_text(json.dumps(result,ensure_ascii=False,indent=2),encoding='utf-8')
(output/'results.json').write_text(
json.dumps(result,ensure_ascii=False,indent=2),encoding='utf-8')
return aggregates
for problem in map(int, args.problems.split(',')):
scenes = []
@@ -71,7 +73,8 @@ def main():
scenes.append((f'{count}-{seed}',count,seed,sources))
for name,count,seed,sources in scenes:
scene_data = dict(sources=[asdict(s) for s in sources], error_seed=seed+2000000)
(output/f'q{problem}-{name}-scene.json').write_text(json.dumps(scene_data),encoding='utf-8')
scene_path = output/f'q{problem}-{name}-scene.json'
scene_path.write_text(json.dumps(scene_data),encoding='utf-8')
for variant in variants:
model = None
cls = BayesLoop if problem==3 else DirectionalExtension
@@ -81,13 +84,15 @@ def main():
model = (Path(f'output/rl/q{problem}_ppo.zip') if variant=='legacy'
else args.model_dir/f'q{problem}_masked.zip')
kwargs = dict(model_path=model, device='cpu')
sim = LocalSimulator(sources,robot_id='RL-AUDIT',spec=referee_spec(config),error_seed=seed+2000000)
sim = LocalSimulator(sources,robot_id='RL-AUDIT',spec=referee_spec(config),
error_seed=seed+2000000)
client = SimulatorClient('RL-AUDIT',config=config,transport=LocalSimTransport(sim))
start = perf_counter()
summary = cls(client,config,**kwargs).run()
events = client.log.action_events()
truth = {s.channel for s in sources}
successful = {e['request']['channel'] for e in events if e['response'].get('clear_result')=='success'}
successful = {e['request']['channel'] for e in events
if e['response'].get('clear_result')=='success'}
seen = set()
repeats = 0
for e in events:
@@ -102,12 +107,15 @@ def main():
and summary['exit_reason']=='user_exit' and not mismatches)
row = dict(case=name,problem=problem,variant=variant,valid=valid,
virtual_s=client.state.virtual_time_s,all_cleared=successful==truth,
measures=summary['measures'],failed_clears=summary['clears']-len(successful),
measures=summary['measures'],
failed_clears=summary['clears']-len(successful),
repeated_measures=repeats,wall_s=perf_counter()-start,
model_sha256=hashlib.sha256(model.read_bytes()).hexdigest() if model else None)
model_sha256=(hashlib.sha256(model.read_bytes()).hexdigest()
if model else None))
rows.append(row)
(output/f'q{problem}-{name}-{variant}.json').write_text(json.dumps(
dict(row=row,summary=summary,events=events,timing_mismatches=mismatches),ensure_ascii=False),encoding='utf-8')
dict(row=row,summary=summary,events=events,timing_mismatches=mismatches),
ensure_ascii=False),encoding='utf-8')
save()
print(json.dumps(row),flush=True)
print(json.dumps(save(),indent=2),flush=True)