sloray/mcm-26-b
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

13

Browse Source
This commit is contained in:
sloray
2026-09-11 17:24:07 +08:00
parent 4699b0d45f
commit 30df22e8b2
240 changed files with 3391 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/mcm26b/planner/rl_loop.py
+4 -2
View File
@@ -41,7 +41,8 @@ class RLBayesLoop(BayesLoop):
point, action = self.rl_policy.select(
region, self.client.state.position, channel, self.config,
problem=3, blocked=[], belief=self.belief.channels[channel],
virtual_time_s=self.client.state.virtual_time_s)
virtual_time_s=self.client.state.virtual_time_s,
receiver_channel=self.client.state.channel)
self.rl_probe_actions.append({
"channel": channel, "action": int(action),
"position": [round(point[0], 3), round(point[1], 3)],
@@ -74,7 +75,8 @@ class RLDirectionalLoop(DirectionalExtension):
region, self.client.state.position, channel, self.config,
problem=4, blocked=blocked,
belief=self.belief.channels[channel],
virtual_time_s=self.client.state.virtual_time_s)
virtual_time_s=self.client.state.virtual_time_s,
receiver_channel=self.client.state.channel)
self.rl_probe_actions.append({
"channel": channel, "action": int(action),
"position": [round(point[0], 3), round(point[1], 3)],
src/mcm26b/rl/env.py
+51 -30
View File
@@ -28,7 +28,7 @@ from ..planner import clear_certificate, fallback_clear_cells, safe_disk_radius
from ..protocol.client import SimulatorClient
from ..protocol.log import SessionLog
from ..protocol.rules import CLEAR_SUCCESS, DIRECTION, NEAR, NO_SIGNAL
from .features import NUM_ACTIONS, OBS_DIM, build_observation, candidate_actions
from .features import NUM_ACTIONS, OBS_DIM, action_mask, build_observation, candidate_actions
class LocalizationEnv(gym.Env):
@@ -44,7 +44,8 @@ class LocalizationEnv(gym.Env):
problem: int = 3, max_episode_steps: int = 16,
directional_probability: float = 0.5,
real_budget_s: float = 3600.0,
error_seed: int | None = None):
error_seed: int | None = None,
handoff_probability: float = 0.35, boundary_probability: float = 0.2):
super().__init__()
if problem not in (3, 4):
raise ValueError("problem 必须是 3 或 4")
@@ -54,6 +55,8 @@ class LocalizationEnv(gym.Env):
self.directional_probability = float(directional_probability)
self.real_budget_s = float(real_budget_s)
self.error_seed = error_seed
self.handoff_probability = handoff_probability
self.boundary_probability = boundary_probability
self.observation_space = spaces.Box(low=-10.0, high=10.0,
shape=(OBS_DIM,), dtype=np.float32)
self.action_space = spaces.Discrete(NUM_ACTIONS)
@@ -89,9 +92,14 @@ class LocalizationEnv(gym.Env):
field = self.config.field
channel = rng.randint(field.channel_min, field.channel_max)
radius = field.omega_radius_m * math.sqrt(rng.random())
edge = rng.random() < self.boundary_probability
if edge:
radius = field.omega_radius_m * rng.uniform(0.98, 1.0)
angle = rng.uniform(0.0, 2 * math.pi)
x, y = radius * math.cos(angle), radius * math.sin(angle)
r_eff = rng.uniform(field.r_eff_min_m, field.r_eff_max_m)
if edge:
r_eff = field.r_eff_min_m
directional = self.problem == 4 and rng.random() < self.directional_probability
heading = rng.uniform(0.0, 360.0) if directional else 0.0
@@ -101,7 +109,7 @@ class LocalizationEnv(gym.Env):
if max_distance < 310.0:
max_distance = 310.0
probe_distance = rng.uniform(300.0, max_distance)
probe_bearing = (heading + rng.uniform(-80.0, 80.0)
probe_bearing = (heading + rng.uniform(-89.9, 89.9)
if directional else rng.uniform(0.0, 360.0))
probe_radians = math.radians(probe_bearing)
probe = (x + probe_distance * math.cos(probe_radians),
@@ -137,6 +145,13 @@ class LocalizationEnv(gym.Env):
self._clear_failed = False
self._measured_points = {self._point_key(probe)}
self._fallback_attempts = 0
# 运行中可能先服务其他频道后再定位本源；模拟这种位置/频道交接。
if self._scene_rng.random() < self.handoff_probability:
angle = self._scene_rng.uniform(0, 2*math.pi)
reach = self.config.field.omega_radius_m * math.sqrt(self._scene_rng.random())
other = self.config.field.channel_min if self._channel != self.config.field.channel_min else self.config.field.channel_max
self._client.measure((reach*math.cos(angle), reach*math.sin(angle)), other)
self._start_virtual_s = self._client.state.virtual_time_s
return True
# near / no_signal 首测属于概率为零的边界或课程构造失败，换场景。
continue
@@ -149,9 +164,12 @@ class LocalizationEnv(gym.Env):
def reset(self, *, seed: int | None = None, options: dict | None = None):
super().reset(seed=seed)
self.close()
if seed is not None:
self._scene_rng = Random(seed)
self._steps = 0
self._localize_steps = 0
self._failure_reason = None
self._terminated = False
self._truncated = False
if not self._make_episode():
@@ -165,8 +183,14 @@ class LocalizationEnv(gym.Env):
raise RuntimeError("首次测向后信念状态缺失候选集")
return build_observation(
region, self._client.state.position, self._channel, self.config,
self.problem, self._blocked, self._belief,
virtual_time_s=self._client.state.virtual_time_s)
self.problem, self._blocked, self._belief.channels[self._channel],
virtual_time_s=self._client.state.virtual_time_s,
receiver_channel=self._client.state.channel)
def action_masks(self):
return action_mask(self._belief.candidate_set(self._channel), self._client.state.position,
self.config, self.problem, self._blocked,
self._belief.channels[self._channel])
def _info(self) -> dict:
source = self._source
@@ -179,6 +203,8 @@ class LocalizationEnv(gym.Env):
"blocked_count": len(self._blocked),
"directional_observed": self._directional_observed,
"fallback_attempts": self._fallback_attempts,
"virtual_cost_s": self._client.state.virtual_time_s - self._start_virtual_s,
"failure_reason": self._failure_reason,
}
def step(self, action):
@@ -199,7 +225,6 @@ class LocalizationEnv(gym.Env):
return self._try_clear(cert, observation=self._observe())
old_virtual = self._client.state.virtual_time_s
old_radius = region.mec()[1]
candidates = candidate_actions(region, self._client.state.position,
self.config, self._blocked)
point = candidates[action]
@@ -207,32 +232,25 @@ class LocalizationEnv(gym.Env):
outcome = self._client.measure(point, self._channel)
self._belief.update_measure(self._channel, point, outcome.result,
outcome.svd_deg)
except Exception:
return self._finish(observation=self._observe(), reward=-20.0,
except Exception as exc:
self._failure_reason = type(exc).__name__ + ': ' + str(exc)
return self._finish(observation=self._observe(), reward=-100.0,
terminated=True, truncated=False)
cost_s = self._client.state.virtual_time_s - old_virtual
reward = -cost_s / 100.0
if self._point_key(point) in self._measured_points:
reward -= 0.75 # 重复测量浪费一次检测+移动，抑制策略回路
else:
self._measured_points.add(self._point_key(point))
self._measured_points.add(self._point_key(point))
new_region = self._belief.candidate_set(self._channel)
new_radius = new_region.mec()[1] if new_region is not None else 0.0
if outcome.result == DIRECTION:
# 密集成型奖励：一次测量把包围半径缩得越多越好。
shrink = max(0.0, old_radius - new_radius)
reward += 3.0 * shrink / max(old_radius, 1.0)
elif outcome.result == NEAR:
reward += 1.5
elif outcome.result == NO_SIGNAL:
if outcome.result == NO_SIGNAL and self.problem == 4:
center = region.mec()[0]
safe = safe_disk_radius(region, self.config)
if safe >= 0 and math.dist(point, center) <= safe + 1e-6:
if not self._directional_observed:
self._directional_observed = True
reward += 2.0 # 首次获得"朝向遮挡"证据（问题4关键分支）
self._blocked.append(point)
self._blocked.append(point)
else:
self._blocked = []
self._localize_steps += 1
self._clear_failed = False
cert = clear_certificate(new_region, self._channel, self.config) \
@@ -241,10 +259,11 @@ class LocalizationEnv(gym.Env):
if cert is not None and cert.certified:
return self._try_clear(cert, observation=observation,
reward=reward)
if self.problem == 4 and len(self._blocked) >= self.config.search.max_blocked_probes:
if self.problem == 4 and (len(self._blocked) >= self.config.search.max_blocked_probes
or self._localize_steps >= self.config.search.max_localize_steps):
return self._fallback_clear(observation=observation, reward=reward)
if self._steps >= self.max_episode_steps:
return self._finish(observation=observation, reward=reward - 2.0,
return self._finish(observation=observation, reward=reward - 100.0,
terminated=False, truncated=True)
return observation, reward, False, False, self._info()
@@ -256,9 +275,9 @@ class LocalizationEnv(gym.Env):
"""
region = self._belief.candidate_set(self._channel)
cells = fallback_clear_cells(region.polygon, self.config)
self._fallback_attempts = len(cells)
self._fallback_attempts = 0
if len(cells) > self.config.search.fallback_max_cells:
return self._finish(observation=observation, reward=reward - 3.0,
return self._finish(observation=observation, reward=reward - 100.0,
terminated=False, truncated=True)
attempts = 0
while cells:
@@ -271,12 +290,12 @@ class LocalizationEnv(gym.Env):
reward += -(self._client.state.virtual_time_s - old_virtual) / 100.0
if outcome.result == CLEAR_SUCCESS:
self._belief.mark_cleared(self._channel)
reward += 8.0
reward += 20.0
self._fallback_attempts = attempts
return self._finish(observation=observation, reward=reward,
terminated=True, truncated=False)
self._fallback_attempts = attempts
return self._finish(observation=observation, reward=reward - 20.0,
return self._finish(observation=observation, reward=reward - 100.0,
terminated=True, truncated=False)
def _try_clear(self, cert, *, observation, reward: float = 0.0):
@@ -286,7 +305,7 @@ class LocalizationEnv(gym.Env):
reward += -cost_s / 100.0
if outcome.result == CLEAR_SUCCESS:
self._belief.mark_cleared(self._channel)
reward += 10.0 + 0.25 * max(0, self.max_episode_steps - self._steps)
reward += 20.0
return self._finish(observation=observation, reward=reward,
terminated=True, truncated=False)
self._clear_failed = True
@@ -302,7 +321,9 @@ class LocalizationEnv(gym.Env):
info = self._info()
info["cleared"] = (self._belief.channels[self._channel].status.value
== "cleared")
self.close()
return observation, float(reward), self._terminated, self._truncated, info
def close(self):
return None
if self._client is not None and self._client.state.entered and not self._client.state.exited:
self._client.exit()
src/mcm26b/rl/features.py
+39 -4
View File
@@ -31,6 +31,7 @@ __all__ = [
"candidate_actions",
"build_observation",
"empty_observation",
"action_mask",
]
@@ -61,8 +62,13 @@ def candidate_actions(region, position: Vec2, config: ProblemConfig,
center, radius = region.mec()
safe = safe_disk_radius(region, config)
base_radius = _probe_radius(radius, safe)
points = candidate_points(tuple(center), base_radius,
_RING_RATIOS, _RING_ANGLES)
# 半径恰为0时也保留固定编号，重复位置由action_mask屏蔽。
points = [tuple(center)]
for fraction in _RING_RATIOS:
for k in range(_RING_ANGLES):
angle = (k + 0.5) * 2 * math.pi / _RING_ANGLES
points.append((center[0] + base_radius*fraction*math.cos(angle),
center[1] + base_radius*fraction*math.sin(angle)))
assert len(points) == 37
base_angle = math.atan2(position[1] - center[1],
@@ -87,6 +93,33 @@ def candidate_actions(region, position: Vec2, config: ProblemConfig,
return points
def action_mask(region, position: Vec2, config: ProblemConfig, problem: int,
blocked=None, belief=None) -> np.ndarray:
"""只依据可见观测屏蔽同地重测、重复候选、越界点。
Q3有距离保证盘时屏蔽盘外探点；Q4允许盘外探点，但绝不把距离保证
当作朝向保证。训练与推理共用此函数，不读取源位置或朝向。
"""
key = lambda p: (round(p[0]*2), round(p[1]*2))
visited = {key(r['position']) for r in getattr(belief, 'measurements', [])}
seen = set()
center, _ = region.mec()
safe = safe_disk_radius(region, config)
result = []
for p in candidate_actions(region, position, config, blocked):
tag = key(p)
valid = (tag not in visited and tag not in seen
and all(math.isfinite(v) and abs(v) <= config.field.coord_abs_limit_m for v in p)
and (problem != 3 or safe < 0 or math.dist(p, center) <= safe + 1e-6))
result.append(valid)
if valid:
seen.add(tag)
if not any(result):
# 退化几何无新候选时保留一个合法动作，由步数/预算边界收尾。
result[0] = True
return np.asarray(result, dtype=bool)
def _last_measurement_features(belief) -> tuple[tuple[float, float, float],
tuple[float, float]]:
"""末次读数的 one-hot + 方向角 sin/cos（无读数取零）。"""
@@ -107,7 +140,8 @@ def _last_measurement_features(belief) -> tuple[tuple[float, float, float],
def build_observation(region, position: Vec2, channel: int, config: ProblemConfig,
problem: int, blocked: list[Vec2] | None = None,
belief=None, virtual_time_s: float = 0.0) -> np.ndarray:
belief=None, virtual_time_s: float = 0.0,
receiver_channel: int | None = None) -> np.ndarray:
"""构造 PPO 观测向量（shape=(401,)，与训练环境逐位一致）。"""
field, search = config.field, config.search
center, radius = region.mec()
@@ -119,7 +153,8 @@ def build_observation(region, position: Vec2, channel: int, config: ProblemConfi
global_features = [
position[0] / 2000.0,
position[1] / 2000.0,
(channel - field.channel_min) / max(field.channel_max - field.channel_min, 1),
((channel if receiver_channel is None else receiver_channel) - field.channel_min)
/ max(field.channel_max - field.channel_min, 1),
center[0] / 2000.0,
center[1] / 2000.0,
radius / 2000.0,
src/mcm26b/rl/policy.py
+26 -5
View File
@@ -8,10 +8,11 @@
from __future__ import annotations
from pathlib import Path
import json
from ..config import ProblemConfig, Vec2
from ._constants import NUM_ACTIONS, OBS_DIM
from .features import build_observation, candidate_actions
from .features import action_mask, build_observation, candidate_actions
class PointSelectionPolicy:
@@ -38,7 +39,22 @@ class PointSelectionPolicy:
except ImportError:
device = "cpu"
self.device = device
self.model = PPO.load(str(path), device=device)
meta_path = path.with_suffix('.json')
self.metadata = json.loads(meta_path.read_text(encoding='utf-8')) if meta_path.exists() else {}
self.feature_version = self.metadata.get('feature_version', 1)
if self.feature_version not in (1, 2):
raise RuntimeError('不支持的RL特征版本')
self.masked = self.metadata.get('algorithm') == 'maskable_ppo'
if self.masked:
from sb3_contrib import MaskablePPO
if self.metadata.get('feature_version') != 2:
raise RuntimeError('动作屏蔽模型的特征版本不匹配')
loader = MaskablePPO
else:
loader = PPO
self.model = loader.load(str(path), device=device)
if not self.masked and type(self.model.policy).__module__.startswith('sb3_contrib'):
raise RuntimeError('动作屏蔽模型缺少匹配的JSON元数据，不能按普通PPO推理')
try:
obs_dim = self.model.observation_space.shape[0]
action_dim = self.model.action_space.n
@@ -53,12 +69,17 @@ class PointSelectionPolicy:
def select(self, region, position: Vec2, channel: int,
config: ProblemConfig, problem: int,
blocked: list[Vec2] | None = None, belief=None,
virtual_time_s: float = 0.0) -> tuple[Vec2, int]:
virtual_time_s: float = 0.0,
receiver_channel: int | None = None) -> tuple[Vec2, int]:
"""返回 (检测点, 候选编号)；确定性 argmax（评估/比赛口径）。"""
if self.metadata.get('problem', problem) != problem:
raise RuntimeError('模型训练问题编号与当前第三/四问不匹配')
observation = build_observation(
region, position, channel, config, problem, blocked, belief,
virtual_time_s=virtual_time_s)
action, _state = self.model.predict(observation, deterministic=True)
virtual_time_s=virtual_time_s,
receiver_channel=receiver_channel if self.feature_version >= 2 else None)
kwargs = {'action_masks': action_mask(region, position, config, problem, blocked, belief)} if self.masked else {}
action, _state = self.model.predict(observation, deterministic=True, **kwargs)
action = int(action)
candidates = candidate_actions(region, position, config, blocked)
return candidates[action], action