强化学习第一版
This commit is contained in:
+33
-6
@@ -30,6 +30,7 @@ from .protocol.log import SessionLog
|
||||
from .protocol.rules import EXIT_USER
|
||||
|
||||
Q3_STRATEGIES = {"grid": OmnidirectionalLoop, "seven": SevenBatch, "bayes": BayesLoop}
|
||||
Q4_STRATEGIES = {"directional": DirectionalExtension}
|
||||
|
||||
|
||||
def _load_referee():
|
||||
@@ -251,16 +252,33 @@ def _run(args) -> int:
|
||||
log = SessionLog(path=Path(args.log)) if args.log else SessionLog()
|
||||
client = SimulatorClient(args.robot_id, config=config, transport=transport, log=log)
|
||||
if args.problem == 4:
|
||||
loop_cls = DirectionalExtension
|
||||
strategy = args.q4_strategy
|
||||
loop_cls = Q4_STRATEGIES.get(strategy)
|
||||
else:
|
||||
loop_cls = Q3_STRATEGIES[args.strategy]
|
||||
strategy = args.strategy
|
||||
loop_cls = Q3_STRATEGIES.get(strategy)
|
||||
try:
|
||||
summary = loop_cls(client, config).run()
|
||||
if strategy == "rl":
|
||||
# 惰性导入：默认运行闭环不加载 numpy/torch/stable-baselines3。
|
||||
from .planner.rl_loop import RLBayesLoop, RLDirectionalLoop
|
||||
loop_cls = RLBayesLoop if args.problem == 3 else RLDirectionalLoop
|
||||
if loop_cls is None:
|
||||
raise RuntimeError(f"未知策略: {strategy}")
|
||||
if strategy == "rl":
|
||||
summary = loop_cls(client, config, model_path=args.rl_model).run()
|
||||
else:
|
||||
summary = loop_cls(client, config).run()
|
||||
except RuntimeError as exc:
|
||||
print(f"RL 策略不可用: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
finally:
|
||||
if judge is not None:
|
||||
_settle_local_judge(judge)
|
||||
summary["robot_id"] = args.robot_id
|
||||
summary["strategy"] = ("directional" if args.problem == 4 else args.strategy)
|
||||
if args.problem == 4:
|
||||
summary["strategy"] = "rl_q4" if strategy == "rl" else strategy
|
||||
else:
|
||||
summary["strategy"] = strategy
|
||||
if real_budget_s is not None:
|
||||
summary["self_test_real_budget_s"] = real_budget_s
|
||||
if args.summary_steps is not None and len(summary.get("steps", [])) > args.summary_steps:
|
||||
@@ -305,10 +323,19 @@ def main(argv=None) -> int:
|
||||
run.add_argument("--log", default=None, help="会话日志 JSONL 输出路径")
|
||||
run.add_argument("--problem", type=int, choices=(3, 4), default=3,
|
||||
help="问题3=全向源，问题4=混合定向源（默认 3）")
|
||||
run.add_argument("--strategy", choices=tuple(Q3_STRATEGIES), default="bayes",
|
||||
run.add_argument("--strategy", choices=tuple(Q3_STRATEGIES) + ("rl",),
|
||||
default="bayes",
|
||||
help="问题3策略：bayes=粒子滤波+信息增益+滚动调度（默认，"
|
||||
"自建消融相对 seven 再降 19.5%%），seven=七点批量，"
|
||||
"grid=49点网格基线；仅 --problem 3 生效")
|
||||
"grid=49点网格基线，rl=PPO 强化学习选点（需先训练模型）；"
|
||||
"仅 --problem 3 生效")
|
||||
run.add_argument("--q4-strategy", choices=tuple(Q4_STRATEGIES) + ("rl",),
|
||||
default="directional",
|
||||
help="问题4策略：directional=49点覆盖+遮挡梯子+后备清除（默认），"
|
||||
"rl=PPO 强化学习选点（需先训练模型）；仅 --problem 4 生效")
|
||||
run.add_argument("--rl-model", default=None,
|
||||
help="RL 模型路径（默认 output/rl/q3_ppo.zip 或 "
|
||||
"output/rl/q4_ppo.zip；训练脚本 scripts/train_rl.py）")
|
||||
run.add_argument("--bearing-margin-deg", type=float, default=0.005,
|
||||
help="示向度舍入裕量（服务器保留两位小数；默认 0.005°）")
|
||||
run.add_argument("--real-budget-s", type=float, default=None,
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
"""P6b 强化学习闭环接入：PPO 学习"下一检测点"选择（问题3/4）。
|
||||
|
||||
边界与保守性声明：
|
||||
- RL 只替换测量点排序（在既有几何候选集内做 argmax），不生成新的
|
||||
空间候选、不修改覆盖证明、不修改清除证书；
|
||||
- 问题3：继承 :class:`BayesLoop`（七点覆盖 + 粒子滤波 + 滚动任务调度），
|
||||
仅 :meth:`_select_point` 改由 PPO 输出候选编号；
|
||||
- 问题4：继承 :class:`DirectionalExtension`（49 点定向覆盖 + 对跖/环探 +
|
||||
20m 有限圆盘覆盖后备清除），仅 :meth:`_next_probe` 改由 PPO 输出；
|
||||
遮挡计数、定向观测与后备清除兜底完全保留；
|
||||
- 训练环境与运行特征同源（mcm26b.rl.features），模型加载时校验
|
||||
观测/动作维度，不匹配即拒绝运行，避免静默错位。
|
||||
|
||||
默认模型位置：output/rl/q3_ppo.zip、output/rl/q4_ppo.zip。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from ..config import ProblemConfig, Vec2
|
||||
from ..rl.policy import PointSelectionPolicy, default_model_path
|
||||
from . import DirectionalExtension
|
||||
from .bayes_loop import BayesLoop
|
||||
|
||||
|
||||
class RLBayesLoop(BayesLoop):
|
||||
"""问题3：贝叶斯闭环骨架 + PPO 定位选点（--strategy rl）。"""
|
||||
|
||||
def __init__(self, client, config: ProblemConfig = ProblemConfig(), *,
|
||||
model_path: str | Path | None = None, device: str = "auto"):
|
||||
super().__init__(client, config)
|
||||
self.rl_model_path = Path(model_path) if model_path else default_model_path(3)
|
||||
self.rl_policy = PointSelectionPolicy(self.rl_model_path, device=device)
|
||||
self.rl_probe_actions: list[dict] = []
|
||||
|
||||
def _select_point(self, channel: int) -> Vec2 | None:
|
||||
region = self.belief.candidate_set(channel)
|
||||
if region is None:
|
||||
return None
|
||||
point, action = self.rl_policy.select(
|
||||
region, self.client.state.position, channel, self.config,
|
||||
problem=3, blocked=[], belief=self.belief.channels[channel],
|
||||
virtual_time_s=self.client.state.virtual_time_s)
|
||||
self.rl_probe_actions.append({
|
||||
"channel": channel, "action": int(action),
|
||||
"position": [round(point[0], 3), round(point[1], 3)],
|
||||
"virtual_time_s": self.client.state.virtual_time_s,
|
||||
})
|
||||
return point
|
||||
|
||||
def _summary(self, info) -> dict:
|
||||
report = super()._summary(info)
|
||||
report["strategy"] = "rl"
|
||||
report["rl_model"] = str(self.rl_model_path)
|
||||
report["rl_device"] = self.rl_policy.device
|
||||
report["rl_probe_actions"] = self.rl_probe_actions
|
||||
return report
|
||||
|
||||
|
||||
class RLDirectionalLoop(DirectionalExtension):
|
||||
"""问题4：定向闭环骨架 + PPO 遮挡梯子选点（--q4-strategy rl）。"""
|
||||
|
||||
def __init__(self, client, config: ProblemConfig = ProblemConfig(), *,
|
||||
model_path: str | Path | None = None, device: str = "auto"):
|
||||
super().__init__(client, config)
|
||||
self.rl_model_path = Path(model_path) if model_path else default_model_path(4)
|
||||
self.rl_policy = PointSelectionPolicy(self.rl_model_path, device=device)
|
||||
self.rl_probe_actions: list[dict] = []
|
||||
|
||||
def _next_probe(self, region, channel: int) -> Vec2:
|
||||
blocked = self.blocked_probes.get(channel, [])
|
||||
point, action = self.rl_policy.select(
|
||||
region, self.client.state.position, channel, self.config,
|
||||
problem=4, blocked=blocked,
|
||||
belief=self.belief.channels[channel],
|
||||
virtual_time_s=self.client.state.virtual_time_s)
|
||||
self.rl_probe_actions.append({
|
||||
"channel": channel, "action": int(action),
|
||||
"position": [round(point[0], 3), round(point[1], 3)],
|
||||
"blocked_count": len(blocked),
|
||||
"virtual_time_s": self.client.state.virtual_time_s,
|
||||
})
|
||||
return point
|
||||
|
||||
def _summary(self, info) -> dict:
|
||||
report = super()._summary(info)
|
||||
report["strategy"] = "rl_q4"
|
||||
report["rl_model"] = str(self.rl_model_path)
|
||||
report["rl_device"] = self.rl_policy.device
|
||||
report["rl_probe_actions"] = self.rl_probe_actions
|
||||
return report
|
||||
@@ -0,0 +1,14 @@
|
||||
"""问题3/4 强化学习扩展（可选依赖组 ``rl``）。
|
||||
|
||||
- features.py：观测/候选动作的纯几何接口（numpy），训练与运行共享；
|
||||
- env.py：单源定位 Gymnasium 训练环境（依赖 referee，仅训练侧加载）；
|
||||
- policy.py：PPO 模型加载与确定性推理（惰性加载 torch/sb3）。
|
||||
|
||||
运行闭环默认不导入 torch/stable-baselines3；只有 CLI 显式选择
|
||||
``--strategy rl``（问题3）或 ``--q4-strategy rl``（问题4）时，
|
||||
``planner.rl_loop`` 才构造 :class:`policy.PointSelectionPolicy`。
|
||||
"""
|
||||
|
||||
from .features import candidate_actions, build_observation
|
||||
|
||||
__all__ = ["candidate_actions", "build_observation"]
|
||||
@@ -0,0 +1,11 @@
|
||||
"""RL 观测/动作空间常量：模型接口的唯一字面量来源。
|
||||
|
||||
布局说明见 features.py；这些数字由训练脚本写入模型接口，
|
||||
改动其中任一值都会使已训练模型失配，须同步重训。
|
||||
"""
|
||||
|
||||
GLOBAL_DIM = 41
|
||||
PER_ACTION_DIM = 6
|
||||
NUM_ACTIONS = 60
|
||||
LADDER_SLOTS = 8
|
||||
OBS_DIM = GLOBAL_DIM + NUM_ACTIONS * PER_ACTION_DIM # 401
|
||||
@@ -0,0 +1,308 @@
|
||||
"""问题3/4 的强化学习训练环境：单源"测向—定位—清除"子任务。
|
||||
|
||||
环境与自建评测机 referee 共享同一边界（内存回环，与真实 HTTP 同协议），
|
||||
观测只来自 :class:`mcm26b.belief.BeliefState` 与协议响应；真值仅在 reset
|
||||
时用于布设课程场景，不进入观测。策略端看到的接口与闭环运行时一致。
|
||||
|
||||
问题抽象：机器狗已获得某频道首次 direction 读数，需要在一组几何可行候选
|
||||
检测点中选择下一个测点，目标是尽快把候选集最小包围圆压到 20m 清除证书
|
||||
阈值并清除成功。问题4额外引入定向源遮挡：保证接收盘内出现 no_signal 是
|
||||
"朝向遮挡"的观测证据，环境记录 blocked 序列并开放环探候选。
|
||||
|
||||
该环境是训练侧组件（依赖 referee + gymnasium），运行时代码不导入本模块；
|
||||
导入依赖由可选组 ``[dependency-groups] rl`` 提供。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from random import Random
|
||||
|
||||
import gymnasium as gym
|
||||
import numpy as np
|
||||
from gymnasium import spaces
|
||||
|
||||
from ..belief import BeliefState
|
||||
from ..config import ProblemConfig
|
||||
from ..planner import clear_certificate, fallback_clear_cells, safe_disk_radius
|
||||
from ..protocol.client import SimulatorClient
|
||||
from ..protocol.log import SessionLog
|
||||
from ..protocol.rules import CLEAR_SUCCESS, DIRECTION, NEAR, NO_SIGNAL
|
||||
from .features import NUM_ACTIONS, OBS_DIM, build_observation, candidate_actions
|
||||
|
||||
|
||||
class LocalizationEnv(gym.Env):
|
||||
"""单源定位清除 Gymnasium 环境。
|
||||
|
||||
problem=3：全向源；problem=4：定向/全向混合（定向概率 0.5，训练分布
|
||||
比题面更稠密以覆盖遮挡分支；正式策略仍由闭环的后备清除兜底）。
|
||||
"""
|
||||
|
||||
metadata = {"render_modes": []}
|
||||
|
||||
def __init__(self, config: ProblemConfig | None = None, *,
|
||||
problem: int = 3, max_episode_steps: int = 16,
|
||||
directional_probability: float = 0.5,
|
||||
real_budget_s: float = 3600.0,
|
||||
error_seed: int | None = None):
|
||||
super().__init__()
|
||||
if problem not in (3, 4):
|
||||
raise ValueError("problem 必须是 3 或 4")
|
||||
self.config = config or ProblemConfig()
|
||||
self.problem = int(problem)
|
||||
self.max_episode_steps = int(max_episode_steps)
|
||||
self.directional_probability = float(directional_probability)
|
||||
self.real_budget_s = float(real_budget_s)
|
||||
self.error_seed = error_seed
|
||||
self.observation_space = spaces.Box(low=-10.0, high=10.0,
|
||||
shape=(OBS_DIM,), dtype=np.float32)
|
||||
self.action_space = spaces.Discrete(NUM_ACTIONS)
|
||||
|
||||
# 惰性导入：本模块只属于训练/评估侧，运行闭环不加载 referee。
|
||||
from referee import LocalSimulator as _RefereeLocalSimulator
|
||||
from referee import Source
|
||||
from ..harness import LocalSimTransport, referee_spec
|
||||
self._Source = Source
|
||||
self._LocalSimulator = _RefereeLocalSimulator
|
||||
self._LocalSimTransport = LocalSimTransport
|
||||
self._referee_spec = referee_spec
|
||||
|
||||
self._scene_rng = Random()
|
||||
self._simulator = None
|
||||
self._client = None
|
||||
self._source = None
|
||||
self._belief: BeliefState | None = None
|
||||
self._channel: int = 1
|
||||
self._blocked: list = []
|
||||
self._directional_observed = False
|
||||
self._clear_failed = False
|
||||
self._steps = 0
|
||||
self._terminated = False
|
||||
self._truncated = False
|
||||
self._measured_points: set = set()
|
||||
self._fallback_attempts = 0
|
||||
|
||||
# ------------------------------------------------------------------ 场景
|
||||
def _make_source_and_probe(self):
|
||||
"""用真值布设一个"首次测向必为 direction"的课程场景（真值不进观测）。"""
|
||||
rng = self._scene_rng
|
||||
field = self.config.field
|
||||
channel = rng.randint(field.channel_min, field.channel_max)
|
||||
radius = field.omega_radius_m * math.sqrt(rng.random())
|
||||
angle = rng.uniform(0.0, 2 * math.pi)
|
||||
x, y = radius * math.cos(angle), radius * math.sin(angle)
|
||||
r_eff = rng.uniform(field.r_eff_min_m, field.r_eff_max_m)
|
||||
directional = self.problem == 4 and rng.random() < self.directional_probability
|
||||
heading = rng.uniform(0.0, 360.0) if directional else 0.0
|
||||
|
||||
# 首次检测点：距离 [300, min(1450, r_eff-20)] 内随机，定向源取朝向
|
||||
# ±80°（远离 90° 边界，保证 direction）。机器狗可到 Ω 外 [题面]。
|
||||
max_distance = max(310.0, min(1450.0, r_eff - 20.0))
|
||||
if max_distance < 310.0:
|
||||
max_distance = 310.0
|
||||
probe_distance = rng.uniform(300.0, max_distance)
|
||||
probe_bearing = (heading + rng.uniform(-80.0, 80.0)
|
||||
if directional else rng.uniform(0.0, 360.0))
|
||||
probe_radians = math.radians(probe_bearing)
|
||||
probe = (x + probe_distance * math.cos(probe_radians),
|
||||
y + probe_distance * math.sin(probe_radians))
|
||||
source = self._Source(channel=channel, x=x, y=y, r_eff_m=r_eff,
|
||||
directional=directional, heading_deg=heading)
|
||||
return source, probe
|
||||
|
||||
def _make_episode(self) -> bool:
|
||||
"""重建模拟器并完成首次测向；返回 False 表示连续未得到 direction。"""
|
||||
for _ in range(20):
|
||||
self._source, probe = self._make_source_and_probe()
|
||||
self._simulator = self._LocalSimulator(
|
||||
[self._source], robot_id="rl-train",
|
||||
spec=self._referee_spec(
|
||||
self.config, max_real_duration_s=self.real_budget_s),
|
||||
error_seed=(self.error_seed if self.error_seed is not None
|
||||
else self._scene_rng.randint(0, 2 ** 31 - 1)),
|
||||
stress_mode=True)
|
||||
self._client = SimulatorClient(
|
||||
"rl-train", config=self.config,
|
||||
transport=self._LocalSimTransport(self._simulator),
|
||||
log=SessionLog())
|
||||
self._client.enter()
|
||||
outcome = self._client.measure(probe, self._source.channel)
|
||||
if outcome.result == DIRECTION:
|
||||
self._channel = self._source.channel
|
||||
self._belief = BeliefState(self.config)
|
||||
self._belief.update_measure(self._channel, probe, outcome.result,
|
||||
outcome.svd_deg)
|
||||
self._blocked = []
|
||||
self._directional_observed = False
|
||||
self._clear_failed = False
|
||||
self._measured_points = {self._point_key(probe)}
|
||||
self._fallback_attempts = 0
|
||||
return True
|
||||
# near / no_signal 首测属于概率为零的边界或课程构造失败，换场景。
|
||||
continue
|
||||
return False
|
||||
|
||||
# ------------------------------------------------------------------ Gym API
|
||||
@staticmethod
|
||||
def _point_key(point) -> tuple[int, int]:
|
||||
return (int(round(point[0] * 2)), int(round(point[1] * 2)))
|
||||
|
||||
def reset(self, *, seed: int | None = None, options: dict | None = None):
|
||||
super().reset(seed=seed)
|
||||
if seed is not None:
|
||||
self._scene_rng = Random(seed)
|
||||
self._steps = 0
|
||||
self._terminated = False
|
||||
self._truncated = False
|
||||
if not self._make_episode():
|
||||
raise RuntimeError("连续 20 次无法构造 direction 首测场景")
|
||||
observation = self._observe()
|
||||
return observation, self._info()
|
||||
|
||||
def _observe(self) -> np.ndarray:
|
||||
region = self._belief.candidate_set(self._channel)
|
||||
if region is None:
|
||||
raise RuntimeError("首次测向后信念状态缺失候选集")
|
||||
return build_observation(
|
||||
region, self._client.state.position, self._channel, self.config,
|
||||
self.problem, self._blocked, self._belief,
|
||||
virtual_time_s=self._client.state.virtual_time_s)
|
||||
|
||||
def _info(self) -> dict:
|
||||
source = self._source
|
||||
return {
|
||||
"channel": self._channel,
|
||||
"source": {"x": source.x, "y": source.y, "r_eff_m": source.r_eff_m,
|
||||
"directional": source.directional,
|
||||
"heading_deg": source.heading_deg},
|
||||
"steps": self._steps,
|
||||
"blocked_count": len(self._blocked),
|
||||
"directional_observed": self._directional_observed,
|
||||
"fallback_attempts": self._fallback_attempts,
|
||||
}
|
||||
|
||||
def step(self, action):
|
||||
if self._terminated or self._truncated:
|
||||
raise RuntimeError("step() 需要在 terminated/truncated 后先 reset()")
|
||||
if not 0 <= int(action) < NUM_ACTIONS:
|
||||
raise ValueError(f"动作 {action} 超出 Discrete({NUM_ACTIONS})")
|
||||
action = int(action)
|
||||
self._steps += 1
|
||||
region = self._belief.candidate_set(self._channel)
|
||||
if region is None:
|
||||
return self._finish(observation=self._observe(), reward=-20.0,
|
||||
terminated=True, truncated=False)
|
||||
|
||||
# 达到清除证书且上次清除失败已被新测量重置：先自动清除（闭环语义）。
|
||||
cert = clear_certificate(region, self._channel, self.config)
|
||||
if cert.certified and not self._clear_failed:
|
||||
return self._try_clear(cert, observation=self._observe())
|
||||
|
||||
old_virtual = self._client.state.virtual_time_s
|
||||
old_radius = region.mec()[1]
|
||||
candidates = candidate_actions(region, self._client.state.position,
|
||||
self.config, self._blocked)
|
||||
point = candidates[action]
|
||||
try:
|
||||
outcome = self._client.measure(point, self._channel)
|
||||
self._belief.update_measure(self._channel, point, outcome.result,
|
||||
outcome.svd_deg)
|
||||
except Exception:
|
||||
return self._finish(observation=self._observe(), reward=-20.0,
|
||||
terminated=True, truncated=False)
|
||||
|
||||
cost_s = self._client.state.virtual_time_s - old_virtual
|
||||
reward = -cost_s / 100.0
|
||||
if self._point_key(point) in self._measured_points:
|
||||
reward -= 0.75 # 重复测量浪费一次检测+移动，抑制策略回路
|
||||
else:
|
||||
self._measured_points.add(self._point_key(point))
|
||||
new_region = self._belief.candidate_set(self._channel)
|
||||
new_radius = new_region.mec()[1] if new_region is not None else 0.0
|
||||
if outcome.result == DIRECTION:
|
||||
# 密集成型奖励：一次测量把包围半径缩得越多越好。
|
||||
shrink = max(0.0, old_radius - new_radius)
|
||||
reward += 3.0 * shrink / max(old_radius, 1.0)
|
||||
elif outcome.result == NEAR:
|
||||
reward += 1.5
|
||||
elif outcome.result == NO_SIGNAL:
|
||||
center = region.mec()[0]
|
||||
safe = safe_disk_radius(region, self.config)
|
||||
if safe >= 0 and math.dist(point, center) <= safe + 1e-6:
|
||||
if not self._directional_observed:
|
||||
self._directional_observed = True
|
||||
reward += 2.0 # 首次获得"朝向遮挡"证据（问题4关键分支）
|
||||
self._blocked.append(point)
|
||||
|
||||
self._clear_failed = False
|
||||
cert = clear_certificate(new_region, self._channel, self.config) \
|
||||
if new_region is not None else None
|
||||
observation = self._observe()
|
||||
if cert is not None and cert.certified:
|
||||
return self._try_clear(cert, observation=observation,
|
||||
reward=reward)
|
||||
if self.problem == 4 and len(self._blocked) >= self.config.search.max_blocked_probes:
|
||||
return self._fallback_clear(observation=observation, reward=reward)
|
||||
if self._steps >= self.max_episode_steps:
|
||||
return self._finish(observation=observation, reward=reward - 2.0,
|
||||
terminated=False, truncated=True)
|
||||
return observation, reward, False, False, self._info()
|
||||
|
||||
def _fallback_clear(self, *, observation, reward: float):
|
||||
"""问题4后备清除语义：遮挡梯子用尽后按 20m 网格覆盖逐格 /clear。
|
||||
|
||||
与 DirectionalExtension._fallback_clear 同口径；训练环境借此让策略
|
||||
学到"梯子耗尽的代价"，运行闭环仍由原后备清除函数执行。
|
||||
"""
|
||||
region = self._belief.candidate_set(self._channel)
|
||||
cells = fallback_clear_cells(region.polygon, self.config)
|
||||
self._fallback_attempts = len(cells)
|
||||
if len(cells) > self.config.search.fallback_max_cells:
|
||||
return self._finish(observation=observation, reward=reward - 3.0,
|
||||
terminated=False, truncated=True)
|
||||
attempts = 0
|
||||
while cells:
|
||||
cell = min(cells, key=lambda c: (
|
||||
math.dist(self._client.state.position, c), c))
|
||||
cells.remove(cell)
|
||||
attempts += 1
|
||||
old_virtual = self._client.state.virtual_time_s
|
||||
outcome = self._client.clear(cell, self._channel)
|
||||
reward += -(self._client.state.virtual_time_s - old_virtual) / 100.0
|
||||
if outcome.result == CLEAR_SUCCESS:
|
||||
self._belief.mark_cleared(self._channel)
|
||||
reward += 8.0
|
||||
self._fallback_attempts = attempts
|
||||
return self._finish(observation=observation, reward=reward,
|
||||
terminated=True, truncated=False)
|
||||
self._fallback_attempts = attempts
|
||||
return self._finish(observation=observation, reward=reward - 20.0,
|
||||
terminated=True, truncated=False)
|
||||
|
||||
def _try_clear(self, cert, *, observation, reward: float = 0.0):
|
||||
old_virtual = self._client.state.virtual_time_s
|
||||
outcome = self._client.clear(cert.center, self._channel)
|
||||
cost_s = self._client.state.virtual_time_s - old_virtual
|
||||
reward += -cost_s / 100.0
|
||||
if outcome.result == CLEAR_SUCCESS:
|
||||
self._belief.mark_cleared(self._channel)
|
||||
reward += 10.0 + 0.25 * max(0, self.max_episode_steps - self._steps)
|
||||
return self._finish(observation=observation, reward=reward,
|
||||
terminated=True, truncated=False)
|
||||
self._clear_failed = True
|
||||
reward -= 1.0
|
||||
if self._steps >= self.max_episode_steps:
|
||||
return self._finish(observation=observation, reward=reward,
|
||||
terminated=False, truncated=True)
|
||||
return observation, reward, False, False, self._info()
|
||||
|
||||
def _finish(self, *, observation, reward, terminated, truncated):
|
||||
self._terminated = bool(terminated)
|
||||
self._truncated = bool(truncated)
|
||||
info = self._info()
|
||||
info["cleared"] = (self._belief.channels[self._channel].status.value
|
||||
== "cleared")
|
||||
return observation, float(reward), self._terminated, self._truncated, info
|
||||
|
||||
def close(self):
|
||||
return None
|
||||
@@ -0,0 +1,173 @@
|
||||
"""RL 状态特征与候选动作生成（问题3/4 共用，运行时与训练环境同源）。
|
||||
|
||||
设计目标：把"下一检测点去哪"这一连续决策转化为固定顺序的离散动作，
|
||||
让 PPO 策略在 GPU 上学习，同时不触碰闭环的保守性证明。候选集合始终由
|
||||
既有几何规则生成（保证接收盘、对称交会、定向遮挡梯子），RL 只负责在
|
||||
这些物理可行候选中排序；证书、覆盖证明与后备清除仍由 planner 层负责。
|
||||
|
||||
观测维度与候选编号一旦训练即成为模型接口，不得随意重排：
|
||||
- 全局特征 41 维（含频道 one-hot 20 维）；
|
||||
- 每个候选 6 维（相对位移、移动耗时、到 MEC 圆心距离、方位、保证接收标记），
|
||||
共 60 × 6 = 360 维；
|
||||
- 总观测 401 维。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
import numpy as np
|
||||
|
||||
from ..config import ProblemConfig, Vec2
|
||||
from ..planner import candidate_points, safe_disk_radius
|
||||
from ._constants import GLOBAL_DIM, LADDER_SLOTS, NUM_ACTIONS, OBS_DIM, PER_ACTION_DIM
|
||||
|
||||
__all__ = [
|
||||
"GLOBAL_DIM",
|
||||
"LADDER_SLOTS",
|
||||
"NUM_ACTIONS",
|
||||
"OBS_DIM",
|
||||
"PER_ACTION_DIM",
|
||||
"candidate_actions",
|
||||
"build_observation",
|
||||
"empty_observation",
|
||||
]
|
||||
|
||||
|
||||
# 候选编号布局（训练/推理共享）：
|
||||
# 0 保证接收盘圆心（或宽候选集 MEC 圆心）
|
||||
# 1..36 保证接收盘 3 环 × 12 方向
|
||||
# 37..51 相对"机器人->MEC 圆心"来向的对称交会候选 3 环 × 5 偏角
|
||||
# 52..59 定向遮挡环探梯子 8 个方向（问题3也保留，作为宽环探索）
|
||||
_RING_RATIOS = (0.4, 0.75, 1.0)
|
||||
_RING_ANGLES = 12
|
||||
_GEO_OFFSETS = (-math.pi / 3, -math.pi / 6, 0.0, math.pi / 6, math.pi / 3)
|
||||
|
||||
|
||||
def _probe_radius(region_radius_m: float, safe_m: float) -> float:
|
||||
"""环候选的基准半径：有保证接收盘用盘半径，否则退化为 MEC 半径。"""
|
||||
if safe_m >= 0.0:
|
||||
return safe_m
|
||||
return max(region_radius_m, 10.0)
|
||||
|
||||
|
||||
def candidate_actions(region, position: Vec2, config: ProblemConfig,
|
||||
blocked: list[Vec2] | None = None) -> list[Vec2]:
|
||||
"""生成固定顺序的 60 个候选检测点。
|
||||
|
||||
参数 region 为 belief.CandidateRegion（任意凸候选集）；blocked 是
|
||||
定向遮挡记录（问题4），问题3 传空列表即可。返回顺序与模型训练时一致。
|
||||
"""
|
||||
center, radius = region.mec()
|
||||
safe = safe_disk_radius(region, config)
|
||||
base_radius = _probe_radius(radius, safe)
|
||||
points = candidate_points(tuple(center), base_radius,
|
||||
_RING_RATIOS, _RING_ANGLES)
|
||||
assert len(points) == 37
|
||||
|
||||
base_angle = math.atan2(position[1] - center[1],
|
||||
position[0] - center[0])
|
||||
for fraction in _RING_RATIOS:
|
||||
for offset in _GEO_OFFSETS:
|
||||
angle = base_angle + offset
|
||||
points.append((center[0] + base_radius * fraction * math.cos(angle),
|
||||
center[1] + base_radius * fraction * math.sin(angle)))
|
||||
|
||||
blocked = list(blocked or [])
|
||||
first = blocked[0] if blocked else None
|
||||
anchor = 0.0
|
||||
if first is not None:
|
||||
anchor = math.atan2(first[1] - center[1], first[0] - center[0])
|
||||
ladder_radius = max(radius, 20.0)
|
||||
for slot in range(LADDER_SLOTS):
|
||||
angle = anchor + slot * math.pi / 4
|
||||
points.append((center[0] + ladder_radius * math.cos(angle),
|
||||
center[1] + ladder_radius * math.sin(angle)))
|
||||
assert len(points) == NUM_ACTIONS
|
||||
return points
|
||||
|
||||
|
||||
def _last_measurement_features(belief) -> tuple[tuple[float, float, float],
|
||||
tuple[float, float]]:
|
||||
"""末次读数的 one-hot + 方向角 sin/cos（无读数取零）。"""
|
||||
readings = getattr(belief, "measurements", None) or []
|
||||
if not readings:
|
||||
return (1.0, 0.0, 0.0), (0.0, 0.0)
|
||||
last = readings[-1]
|
||||
result = last.get("result")
|
||||
no_signal = 1.0 if result == "no_signal" else 0.0
|
||||
direction = 1.0 if result == "direction" else 0.0
|
||||
near = 1.0 if result == "near" else 0.0
|
||||
svd = last.get("svd_deg")
|
||||
if direction and svd is not None:
|
||||
radians = math.radians(float(svd))
|
||||
return (no_signal, direction, near), (math.sin(radians), math.cos(radians))
|
||||
return (no_signal, direction, near), (0.0, 0.0)
|
||||
|
||||
|
||||
def build_observation(region, position: Vec2, channel: int, config: ProblemConfig,
|
||||
problem: int, blocked: list[Vec2] | None = None,
|
||||
belief=None, virtual_time_s: float = 0.0) -> np.ndarray:
|
||||
"""构造 PPO 观测向量（shape=(401,)，与训练环境逐位一致）。"""
|
||||
field, search = config.field, config.search
|
||||
center, radius = region.mec()
|
||||
safe = safe_disk_radius(region, config)
|
||||
readings, svd_pair = _last_measurement_features(belief)
|
||||
xs = [p[0] for p in region.polygon]
|
||||
ys = [p[1] for p in region.polygon]
|
||||
|
||||
global_features = [
|
||||
position[0] / 2000.0,
|
||||
position[1] / 2000.0,
|
||||
(channel - field.channel_min) / max(field.channel_max - field.channel_min, 1),
|
||||
center[0] / 2000.0,
|
||||
center[1] / 2000.0,
|
||||
radius / 2000.0,
|
||||
max(-1.0, min(1.0, safe / 1000.0)),
|
||||
min(len(getattr(belief, "measurements", []) if belief is not None else []),
|
||||
12) / 12.0,
|
||||
min(float(virtual_time_s or 0.0), field.max_virtual_duration_s)
|
||||
/ field.max_virtual_duration_s,
|
||||
# 证书信息由运行时的 clear_certificate 判定传入；训练环境用 region.mec
|
||||
# 的保守近似：radius + slack <= 20 - margin 即可清除。
|
||||
1.0 if radius + region.slack_m <= field.clear_radius_m - search.clear_margin_m else 0.0,
|
||||
float(problem == 4),
|
||||
min(len(blocked or []) / max(search.max_blocked_probes, 1), 1.0),
|
||||
(blocked[0][0] / 2000.0) if blocked else 0.0,
|
||||
(blocked[0][1] / 2000.0) if blocked else 0.0,
|
||||
readings[0],
|
||||
readings[1],
|
||||
readings[2],
|
||||
svd_pair[0],
|
||||
svd_pair[1],
|
||||
(max(xs) - min(xs)) / 4000.0,
|
||||
(max(ys) - min(ys)) / 4000.0,
|
||||
]
|
||||
one_hot = np.zeros(field.channel_max - field.channel_min + 1, dtype=np.float32)
|
||||
one_hot[int(channel) - field.channel_min] = 1.0
|
||||
global_features.extend(one_hot.tolist())
|
||||
assert len(global_features) == GLOBAL_DIM
|
||||
|
||||
action_features: list[float] = []
|
||||
for point in candidate_actions(region, position, config, blocked):
|
||||
dx, dy = point[0] - position[0], point[1] - position[1]
|
||||
distance = math.hypot(dx, dy)
|
||||
action_features.extend([
|
||||
dx / 2000.0,
|
||||
dy / 2000.0,
|
||||
min(distance / (config.cost.speed_mps * 1500.0), 2.0), # 移动耗时 / 1500s
|
||||
math.hypot(point[0] - center[0], point[1] - center[1]) / 2000.0,
|
||||
(math.degrees(math.atan2(dy, dx)) % 360.0) / 360.0,
|
||||
1.0 if safe >= 0 and math.hypot(point[0] - center[0],
|
||||
point[1] - center[1]) <= safe + 1e-6 else 0.0,
|
||||
])
|
||||
assert len(action_features) == NUM_ACTIONS * PER_ACTION_DIM
|
||||
|
||||
observation = np.asarray(global_features + action_features, dtype=np.float32)
|
||||
assert observation.shape == (OBS_DIM,)
|
||||
return observation
|
||||
|
||||
|
||||
def empty_observation() -> np.ndarray:
|
||||
"""依赖缺失时的零观测（仅用于错误提示路径，不进入训练数据）。"""
|
||||
return np.zeros(OBS_DIM, dtype=np.float32)
|
||||
@@ -0,0 +1,71 @@
|
||||
"""RL 选点策略加载与推理（运行时接入 planner 的唯一入口）。
|
||||
|
||||
运行闭环对依赖保持惰性：只有显式选择 ``--strategy rl`` / ``--q4-strategy rl``
|
||||
时才导入 stable_baselines3/torch。模型文件由 scripts/train_rl.py 生成，
|
||||
观测/动作布局与训练环境共享 features 模块。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from ..config import ProblemConfig, Vec2
|
||||
from ._constants import NUM_ACTIONS, OBS_DIM
|
||||
from .features import build_observation, candidate_actions
|
||||
|
||||
|
||||
class PointSelectionPolicy:
|
||||
"""加载 PPO 模型，把当前候选集映射到 60 个候选检测点之一。"""
|
||||
|
||||
def __init__(self, model_path: str | Path, *, device: str = "auto"):
|
||||
path = Path(model_path)
|
||||
if not path.is_file():
|
||||
raise RuntimeError(
|
||||
f"未找到 RL 模型 {path}。请先安装 rl 依赖组并训练：\n"
|
||||
f" uv sync --group rl --group dev\n"
|
||||
f" uv run python scripts/train_rl.py --problem 3 --timesteps 120000\n"
|
||||
f" uv run python scripts/train_rl.py --problem 4 --timesteps 40000")
|
||||
try:
|
||||
from stable_baselines3 import PPO
|
||||
except ImportError as exc:
|
||||
raise RuntimeError(
|
||||
"缺少 RL 运行依赖（stable-baselines3）。请执行 "
|
||||
"`uv sync --group rl` 后重试。") from exc
|
||||
if device == "auto":
|
||||
try:
|
||||
import torch
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
except ImportError:
|
||||
device = "cpu"
|
||||
self.device = device
|
||||
self.model = PPO.load(str(path), device=device)
|
||||
try:
|
||||
obs_dim = self.model.observation_space.shape[0]
|
||||
action_dim = self.model.action_space.n
|
||||
except Exception as exc:
|
||||
raise RuntimeError(f"模型空间与 RL 接口不兼容: {path}") from exc
|
||||
if obs_dim != OBS_DIM or action_dim != NUM_ACTIONS:
|
||||
raise RuntimeError(
|
||||
f"模型接口失配: 期望 obs={OBS_DIM}/act={NUM_ACTIONS}，"
|
||||
f"实际 obs={obs_dim}/act={action_dim}。请用当前版本重新训练。")
|
||||
self.path = path
|
||||
|
||||
def select(self, region, position: Vec2, channel: int,
|
||||
config: ProblemConfig, problem: int,
|
||||
blocked: list[Vec2] | None = None, belief=None,
|
||||
virtual_time_s: float = 0.0) -> tuple[Vec2, int]:
|
||||
"""返回 (检测点, 候选编号)；确定性 argmax（评估/比赛口径）。"""
|
||||
observation = build_observation(
|
||||
region, position, channel, config, problem, blocked, belief,
|
||||
virtual_time_s=virtual_time_s)
|
||||
action, _state = self.model.predict(observation, deterministic=True)
|
||||
action = int(action)
|
||||
candidates = candidate_actions(region, position, config, blocked)
|
||||
return candidates[action], action
|
||||
|
||||
|
||||
def default_model_path(problem: int) -> Path:
|
||||
"""各问题的默认模型位置（训练脚本输出口径）。"""
|
||||
if problem not in (3, 4):
|
||||
raise ValueError("problem 必须是 3 或 4")
|
||||
return Path("output/rl") / f"q{problem}_ppo.zip"
|
||||
Reference in New Issue
Block a user