feat: 链路控制、延迟校准与长测工具,补全文档
构建客户端 / 检查(类型 + 单元测试) (push) Canceled after 0s
构建客户端 / Linux x64(deb + AppImage) (push) Canceled after 0s
构建客户端 / Windows x64(NSIS 安装包) (push) Canceled after 0s
构建客户端 / macOS 通用(.app + .dmg) (push) Canceled after 0s

- 固件:新增 link_control / stream_control,覆盖运行中 Wi-Fi 断线恢复、取帧节拍、
  温控与质量/帧率减载;UDP 发送增加有界重试预算与帧期限,PSRAM JPEG 直发减少拷贝。
- 协议:重组器严格校验分片布局,统计未收齐、超时与完全缺失的帧,处理帧号回绕与设备重启。
- 桌面端:视频改 latest-value 通道并前置帧时间标记,拒绝积压和慢解码的过期画面;新增
  延迟标记页与 --latency-report 校准,完善录制、服务端与退出流程。
- 文档与脚本:新增硬件、开发、延迟、功耗、实时性、稳定性文档和改动记录,重写根 README;
  新增构建、性能、延迟与串口长测脚本,以及长测客户端脚本。
- 构建:固定 esp-1.97.0.0 工具链,分区表随构建复制,更新 sdkconfig 与忽略规则。
This commit is contained in:
2026-10-19 00:19:07 +08:00
parent 4794bed2cf
commit cc6b82d7ec
60 changed files with 4347 additions and 655 deletions
+8 -2
View File
@@ -23,6 +23,7 @@ import { useEffect, useRef, type ReactNode } from "react";
import { streamClient } from "../lib/store";
import { withAlpha } from "../lib/format";
import type { CrosshairConfig, GridConfig, HitFeedbackConfig } from "../lib/types";
import type { FrameInfo } from "../lib/stream";
export interface ViewTransform {
zoom: number;
@@ -61,6 +62,8 @@ interface Props {
hitFeedback: HitFeedbackConfig;
showZoom: boolean;
children?: ReactNode;
/** Only used by the opt-in screen/camera calibration page. */
onFramePainted?: (canvas:HTMLCanvasElement,frame:FrameInfo,drawMs:number)=>void;
}
export function VideoSurface({
@@ -77,12 +80,13 @@ export function VideoSurface({
hitFeedback,
showZoom,
children,
onFramePainted,
}: Props) {
const canvasRef = useRef<HTMLCanvasElement | null>(null);
// 绘制循环不随 props 重建,所以最新一份 props 放在 ref 里给它读。
const props = useRef({ camera, mirrored, crosshair, grid, view, hit, hitFeedback });
props.current = { camera, mirrored, crosshair, grid, view, hit, hitFeedback };
const props = useRef({ camera, mirrored, crosshair, grid, view, hit, hitFeedback,onFramePainted });
props.current = { camera, mirrored, crosshair, grid, view, hit, hitFeedback,onFramePainted };
const drag = useRef<{ x: number; y: number; originX: number; originY: number } | null>(null);
const dirty = useRef(true);
@@ -127,6 +131,7 @@ export function VideoSurface({
const current = props.current;
const frame = streamClient().latest(current.camera);
const newFrame=frame && frame.frameId!==lastFrameId.current;
if (frame && frame.frameId !== lastFrameId.current) {
lastFrameId.current = frame.frameId;
@@ -145,6 +150,7 @@ export function VideoSurface({
dirty.current = false;
draw(canvas, current, frame, timestamp);
if(newFrame && current.onFramePainted) current.onFramePainted(canvas,frame,performance.now());
};
raf = requestAnimationFrame(loop);
+101
View File
@@ -0,0 +1,101 @@
/** A screen marker observed through the real camera. No device clock sync. */
export type Point = { x: number; y: number };
export const CORNERS = [[255,32,32],[32,255,32],[32,96,255],[255,32,255]] as const;
export const MARKER_BITS = 16;
export function markerBits(counter: number): number[] {
const gray=(counter&4095)^((counter&4095)>>>1);
return [1,0,1,0,...Array.from({length:12},(_,i)=>(gray>>>(11-i))&1)];
}
export function drawMarker(ctx: CanvasRenderingContext2D, counter:number) {
const w=ctx.canvas.width,h=ctx.canvas.height;
ctx.fillStyle='#30363e';ctx.fillRect(0,0,w,h);
// Static coloured detail keeps the target from being an unrealistically tiny JPEG.
for(let y=0;y<8;y++) for(let x=0;x<12;x++) {
ctx.fillStyle=`hsl(${(x*31+y*47)%360} 10% ${20+(x+y)%3*7}%)`;
ctx.fillRect(x*w/12,y*h/8,w/12,h/8);
}
[[.06,.08],[.94,.08],[.94,.92],[.06,.92]].forEach(([x,y],i)=>{
ctx.fillStyle=`rgb(${CORNERS[i].join(',')})`;
ctx.fillRect(x*w-.035*w,y*h-.045*h,.07*w,.09*h);
});
const bits=markerBits(counter);
ctx.fillStyle='#222';ctx.fillRect(.12*w,.34*h,.76*w,.32*h);
bits.forEach((bit,i)=>{
const x=(.14+.72*i/MARKER_BITS)*w,cw=.72*w/MARKER_BITS;
ctx.fillStyle=bit?'#fff':'#000';ctx.fillRect(x,.38*h,cw,.10*h);
ctx.fillStyle=bit?'#000':'#fff';ctx.fillRect(x,.52*h,cw,.10*h);
});
}
function colour(r:number,g:number,b:number):number {
// Screen primaries can clip and bloom through exposure/white balance.
// Absolute channel separation admits pastel green/cyan while excluding
// the low-saturation background (which can otherwise look more blue).
if(r>150 && b>150 && r-g>40 && b-g>40 && Math.abs(r-b)<100) return 4;
if(r>150 && r-g>40 && r-b>40) return 1;
if(g>150 && g-r>35 && g-b>30) return 2;
if(b>150 && b>r*1.7 && b-g>20) return 3;
return 0;
}
function project(corners:Point[],u:number,v:number):Point|null {
const [p0,p1,p2,p3]=corners;
const dx1=p1.x-p2.x, dx2=p3.x-p2.x, dx3=p0.x-p1.x+p2.x-p3.x;
const dy1=p1.y-p2.y, dy2=p3.y-p2.y, dy3=p0.y-p1.y+p2.y-p3.y;
const det=dx1*dy2-dx2*dy1;
if(Math.abs(det)<1e-6) return null;
const g=(dx3*dy2-dx2*dy3)/det, h=(dx1*dy3-dx3*dy1)/det;
const den=g*u+h*v+1;
if(Math.abs(den)<.01) return null;
return {x:((p1.x-p0.x+g*p1.x)*u+(p3.x-p0.x+h*p3.x)*v+p0.x)/den,
y:((p1.y-p0.y+g*p1.y)*u+(p3.y-p0.y+h*p3.y)*v+p0.y)/den};
}
export function decodeMarker(pixels:Uint8ClampedArray,w:number,h:number):{counter:number;corners:Point[]}|null {
const labels=new Uint8Array(w*h);
for(let i=0;i<labels.length;i++) labels[i]=colour(pixels[i*4],pixels[i*4+1],pixels[i*4+2]);
const best=Array.from({length:4},()=>({count:0,x:0,y:0}));
const queue=new Int32Array(w*h);
for(let start=0;start<labels.length;start++) {
const cls=labels[start]; if(!cls) continue;
let head=0,tail=1,count=0,sx=0,sy=0;
queue[0]=start;labels[start]=0;
while(head<tail) {
const i=queue[head++],x=i%w,y=Math.floor(i/w);
count++;sx+=x;sy+=y;
for(const n of [x>0?i-1:-1,x<w-1?i+1:-1,y>0?i-w:-1,y<h-1?i+w:-1]) {
if(n>=0 && labels[n]===cls) { labels[n]=0;queue[tail++]=n; }
}
}
if(count>=6 && count<w*h/6 && count>best[cls-1].count) best[cls-1]={count,x:sx/count,y:sy/count};
}
if(best.some(b=>!b.count)) return null;
const corners=best.map(({x,y})=>({x,y}));
const area=Math.abs(corners.reduce((a,p,i)=>a+p.x*corners[(i+1)%4].y-p.y*corners[(i+1)%4].x,0))/2;
if(area<1500) return null;
const luma=(x:number,y:number)=>{
let total=0,count=0;
for(let dy=-1;dy<=1;dy++) for(let dx=-1;dx<=1;dx++) {
const xx=Math.round(x)+dx,yy=Math.round(y)+dy;
if(xx<0||xx>=w||yy<0||yy>=h) continue;
const i=(yy*w+xx)*4;total+=(pixels[i]+pixels[i+1]+pixels[i+2])/3;count++;
}
return count?total/count:NaN;
};
const bits:number[]=[];
for(let i=0;i<MARKER_BITS;i++) {
const x=.14+.72*(i+.5)/MARKER_BITS;
const top=project(corners,(x-.06)/.88,(.43-.08)/.84);
const bottom=project(corners,(x-.06)/.88,(.57-.08)/.84);
if(!top||!bottom) return null;
const delta=luma(top.x,top.y)-luma(bottom.x,bottom.y);
if(!Number.isFinite(delta)||Math.abs(delta)<60) return null;
bits.push(delta>0?1:0);
}
if(bits.slice(0,4).join('')!=='1010') return null;
let gray=bits.slice(4).reduce((n,b)=>(n<<1)|b,0),counter=gray;
while(gray>>>=1) counter^=gray;
return {counter,corners};
}
+2
View File
@@ -76,6 +76,8 @@ export function currentStore(): AppStore {
function handleMessage(msg: WireMsg) {
switch (msg.t) {
case "frameTiming":
return; // Consumed by StreamClient before reaching application state.
case "snapshot": {
const { t: _tag, ...snapshot } = msg;
emit({ snapshot: snapshot as Snapshot });
+82 -20
View File
@@ -28,7 +28,7 @@ export interface FrameInfo {
width: number;
height: number;
bitmap: ImageBitmap;
/** 收到的时刻(`performance.now()`),界面用来显示端到端延迟。 */
/** 本地完成解码的时刻(`performance.now()`),用于判断帧的新旧。 */
receivedAt: number;
}
@@ -39,6 +39,13 @@ interface Handlers {
/** 断线重连的退避上限。 */
const MAX_BACKOFF_MS = 3000;
const MAX_LOCAL_VIDEO_AGE_MS = 100;
interface PendingFrame {
bytes: ArrayBuffer;
arrivedAt: number;
publishedUnixMs?: number;
}
export class StreamClient {
private ws: WebSocket | null = null;
@@ -48,14 +55,19 @@ export class StreamClient {
private retryTimer: number | null = null;
private backoff = 250;
private closed = false;
private generation = 0;
private readonly pending = new Map<number, PendingFrame>();
private readonly frameTimes = new Map<number, { frameId: number; publishedUnixMs: number }>();
private readonly decoding = new Map<number, number>();
/** 统计:收到多少帧、解码失败多少张。界面用它区分"链路没数据"和"数据坏了"。 */
readonly counters = { frames: 0, decodeFailures: 0, reconnects: 0 };
readonly counters = { frames: 0, decodeFailures: 0, reconnects: 0, skippedDecodes: 0, staleFrames: 0 };
constructor(handlers: Handlers) {
constructor(handlers: Handlers, private readonly now = () => performance.now(), private readonly wallNow = () => Date.now()) {
this.handlers = handlers;
}
connect(url: string) {
this.close();
this.url = url;
this.closed = false;
this.open();
@@ -76,11 +88,17 @@ export class StreamClient {
this.ws = socket;
socket.onopen = () => {
if (this.ws !== socket) return;
this.backoff = 250;
this.handlers.onConnection(true);
};
socket.onclose = () => {
if (this.ws !== socket) return;
this.generation += 1;
this.pending.clear();
this.frameTimes.clear();
this.decoding.clear();
this.handlers.onConnection(false);
this.scheduleReconnect();
};
@@ -90,9 +108,12 @@ export class StreamClient {
};
socket.onmessage = (event) => {
if (this.ws !== socket) return;
if (typeof event.data === "string") {
try {
this.handlers.onMessage(JSON.parse(event.data) as WireMsg);
const message = JSON.parse(event.data) as WireMsg;
if (message.t === "frameTiming") this.onTiming(message);
else this.handlers.onMessage(message);
} catch {
// 前端解析不了的消息只可能是版本不匹配,忽略比崩掉好。
}
@@ -103,31 +124,66 @@ export class StreamClient {
};
}
private onTiming(message: { camera: number; frameId: number; publishedUnixMs: number }) {
if (Number.isFinite(message.publishedUnixMs) && message.publishedUnixMs > 0) {
this.frameTimes.set(message.camera, message);
}
}
private expired(frame: PendingFrame): boolean {
return this.now() - frame.arrivedAt >= MAX_LOCAL_VIDEO_AGE_MS ||
(frame.publishedUnixMs !== undefined && this.wallNow() - frame.publishedUnixMs >= MAX_LOCAL_VIDEO_AGE_MS);
}
private async onBinary(buffer: ArrayBuffer) {
if (buffer.byteLength < FRAME_HEADER_LEN) return;
if (this.closed || buffer.byteLength < FRAME_HEADER_LEN) return;
const view = new DataView(buffer);
const camera = view.getUint32(0, true);
const frameId = view.getUint32(4, true);
const width = view.getUint16(8, true);
const height = view.getUint16(10, true);
const length = view.getUint32(12, true);
if (buffer.byteLength < FRAME_HEADER_LEN + length) return;
if (buffer.byteLength !== FRAME_HEADER_LEN + length) return;
this.counters.frames += 1;
const timing = this.frameTimes.get(camera);
const received: PendingFrame = { bytes: buffer, arrivedAt: this.now(),
publishedUnixMs: timing?.frameId === view.getUint32(4, true) ? timing.publishedUnixMs : undefined };
if (this.expired(received)) { this.counters.staleFrames += 1; return; }
if (this.pending.has(camera)) this.counters.skippedDecodes += 1;
this.pending.set(camera, received);
if (this.decoding.has(camera)) return;
const jpeg = new Uint8Array(buffer, FRAME_HEADER_LEN, length);
let bitmap: ImageBitmap;
const generation = this.generation;
this.decoding.set(camera, generation);
try {
bitmap = await createImageBitmap(new Blob([jpeg], { type: "image/jpeg" }));
} catch {
this.counters.decodeFailures += 1;
return;
while (!this.closed && this.generation === generation) {
const next = this.pending.get(camera);
if (!next) break;
this.pending.delete(camera);
if (this.expired(next)) { this.counters.staleFrames += 1; continue; }
const header = new DataView(next.bytes);
const jpeg = new Uint8Array(next.bytes, FRAME_HEADER_LEN);
let bitmap: ImageBitmap;
try {
bitmap = await createImageBitmap(new Blob([jpeg], { type: "image/jpeg" }));
} catch {
this.counters.decodeFailures += 1;
continue;
}
if (this.closed || this.generation !== generation) {
bitmap.close();
break;
}
if (this.expired(next)) {
bitmap.close();
this.counters.staleFrames += 1;
continue;
}
this.store(camera, header.getUint32(4, true), header.getUint16(8, true), header.getUint16(10, true), bitmap);
}
} finally {
if (this.decoding.get(camera) === generation) this.decoding.delete(camera);
}
this.store(camera, frameId, width, height, bitmap);
}
/**
@@ -149,7 +205,9 @@ export class StreamClient {
const current = this.frames.get(camera);
// 带符号差处理 u32 回绕。
if (current && ((frameId - current.frameId) | 0) <= 0) {
// After a quiet interval the board may have rebooted and restarted its
// frame counter. An older in-flight decode cannot hold the display forever.
if (current && ((frameId - current.frameId) | 0) <= 0 && this.now() - current.receivedAt < 1000) {
bitmap.close();
return;
}
@@ -162,7 +220,7 @@ export class StreamClient {
width,
height,
bitmap,
receivedAt: performance.now(),
receivedAt: this.now(),
});
}
@@ -197,6 +255,10 @@ export class StreamClient {
close() {
this.closed = true;
this.generation += 1;
this.pending.clear();
this.frameTimes.clear();
this.decoding.clear();
if (this.retryTimer !== null) {
window.clearTimeout(this.retryTimer);
+1
View File
@@ -233,6 +233,7 @@ export type WireMsg =
| ({ t: "snapshot" } & Snapshot)
| ({ t: "hit" } & HitEvent)
| { t: "note"; message: string }
| { t: "frameTiming"; camera: number; frameId: number; publishedUnixMs: number }
| { t: "confirmQuit" };
/** 二进制视频帧的头部长度,与 `server::FRAME_HEADER_LEN` 一致。 */
+6 -1
View File
@@ -1,6 +1,8 @@
import { createRoot } from "react-dom/client";
import App from "./App";
import { LatencyPage } from './pages/LatencyPage';
import { invoke } from '@tauri-apps/api/core';
import "./styles/global.css";
const container = document.getElementById("root");
@@ -12,4 +14,7 @@ if (!container) {
// 不用 StrictMode:这个界面的关键路径是命令式的(WebSocket、requestAnimationFrame 绘制
// 循环、canvas 尺寸)。StrictMode 在开发下会把每个 effect 跑两遍,于是连接与绘制循环会
// 短暂地建两份——排查起来的噪音大于它带来的收益。生产构建的行为与这里一致。
createRoot(container).render(<App />);
const root=createRoot(container);
if('__TAURI_INTERNALS__' in window) {
void invoke<boolean>('latency_enabled').then(enabled=>root.render(enabled?<LatencyPage/>:<App/>),()=>root.render(<App/>));
} else root.render(<App/>);
+101
View File
@@ -0,0 +1,101 @@
import { useEffect,useRef,useState } from 'react';
import { invoke } from '@tauri-apps/api/core';
import { api } from '../lib/ipc';
import { startStream,useAppStore } from '../lib/store';
import { decodeMarker,drawMarker } from '../lib/latency-marker';
import { VideoSurface,IDENTITY_VIEW } from '../components/VideoSurface';
import type { FrameInfo } from '../lib/stream';
type Stamp={draw:number;next:number|null;sampled:boolean};
const feedback={flash:false,shake:false,marker:false,intensity:0};
export function LatencyPage() {
const {snapshot,connected}=useAppStore();
const [camera,setCamera]=useState<number|null>(null);
const [status,setStatus]=useState('等待摄像头与屏幕标记');
const [summary,setSummary]=useState({samples:0,p50:0,p95:0,refresh:0});
const target=useRef<HTMLCanvasElement>(null);
const history=useRef(new Map<number,Stamp>());
const scratch=useRef<HTMLCanvasElement|null>(null);
const values=useRef<number[]>([]);
const lastSample=useRef(-Infinity);
const refresh=useRef(16.7);
const active=useRef(true);
useEffect(()=>{
let client:ReturnType<typeof startStream>|null=null,disposed=false;
void api.streamInfo().then(info=>{if(!disposed)client=startStream(info.url);}).catch(e=>setStatus(String(e)));
return()=>{disposed=true;client?.close();};
},[]);
useEffect(()=>{
if(camera===null && snapshot) {
const found=snapshot.cameras.find(c=>c.online);
if(found)setCamera(found.id);
}
},[snapshot,camera]);
useEffect(()=>{
active.current=true;
const canvas=target.current;if(!canvas)return;
const ctx=canvas.getContext('2d');if(!ctx)return;
let raf=0,counter=0,previous:Stamp|null=null,last=0,lastMarker=-Infinity;
const loop=(now:number)=>{
if(previous){previous.next=now;previous=null;}
if(last && now-last<50)refresh.current=refresh.current*.9+(now-last)*.1;
last=now;
// A 180 Hz display can change several IDs within one camera exposure.
// Hold each event for 50 ms, and record only its first decodable echo.
if(now-lastMarker>=50){
lastMarker=now;counter=(counter+1)&4095;
drawMarker(ctx,counter);
const stamp={draw:performance.now(),next:null,sampled:false};
history.current.set(counter,stamp);previous=stamp;
}
// Only the most recent five seconds count, including counter wrap.
for(const [id,value] of history.current) if(now-value.draw>5000)history.current.delete(id);
raf=requestAnimationFrame(loop);
};
raf=requestAnimationFrame(loop);
return()=>{active.current=false;cancelAnimationFrame(raf);};
},[]);
const painted=(canvas:HTMLCanvasElement,frame:FrameInfo,drawMs:number)=>{
if(drawMs-lastSample.current<25)return;
lastSample.current=drawMs;
const started=performance.now();
const small=scratch.current??(scratch.current=document.createElement('canvas'));
small.width=320;small.height=240;
const ctx=small.getContext('2d',{willReadFrequently:true});if(!ctx)return;
// Read the canvas after the production renderer has painted the frame.
ctx.drawImage(canvas,0,0,320,240);
const marker=decodeMarker(ctx.getImageData(0,0,320,240).data,320,240);
const analysisMs=performance.now()-started;
if(!marker){setStatus('未识别到四角标记:请让左侧标记完整入镜并尽量拍大');return;}
const source=history.current.get(marker.counter);
if(!source||source.next===null||source.draw>drawMs){setStatus('等待有效的同一会话标记');return;}
if(source.sampled)return;
source.sampled=true;
requestAnimationFrame(nextRafMs=>{
if(!active.current)return;
const lowerMs=Math.max(0,drawMs-source.next!);
const upperMs=nextRafMs-source.draw;
if(upperMs<lowerMs||upperMs>5000)return;
const value=(lowerMs+upperMs)/2;
values.current.push(value);
const sorted=[...values.current].sort((a,b)=>a-b);
setStatus('标记已锁定,正在记录真实相机 → 客户端画布链路');
setSummary({samples:sorted.length,p50:sorted[Math.floor((sorted.length-1)*.5)],p95:sorted[Math.floor((sorted.length-1)*.95)],refresh:refresh.current});
void invoke('latency_sample',{sample:{frameId:frame.frameId,counter:marker.counter,sourceDrawMs:source.draw,
drawMs,nextRafMs,lowerMs,upperMs,decodeToDrawMs:drawMs-frame.receivedAt,analysisMs}}).catch(e=>setStatus(`报告写入失败:${String(e)}`));
});
};
return <main style={{height:'100vh',display:'flex',flexDirection:'column',padding:20,gap:12,background:'#111820',color:'#f3f6fa'}}>
<h1 style={{fontSize:24}}>屏幕 → 摄像头 → 客户端绘制延迟校准</h1>
<p>将摄像头固定对准左侧标记,完整拍到四个彩色角块。不要拍右侧回显区域;保持窗口位置不动。</p>
<div style={{display:'grid',gridTemplateColumns:'1fr 1fr',gap:16,flex:1,minHeight:0}}>
<canvas ref={target} width={800} height={600} style={{width:'100%',height:'100%',objectFit:'contain'}}/>
<div style={{position:'relative',minHeight:0}}><VideoSurface camera={camera} emptyText="等待真实摄像头" mirrored={false}
crosshair={null} grid={null} view={IDENTITY_VIEW} onView={()=>{}} hitFeedback={feedback} showZoom={false} onFramePainted={painted}/></div>
</div>
<p>{connected?'已连接':'连接中'} · {status}</p>
<p style={{fontSize:22}}>样本 {summary.samples} · P50 {summary.p50.toFixed(1)} ms · P95 {summary.p95.toFixed(1)} ms</p>
<p>每 50 ms 更新标记,记录首次可解码回显,包含相机取样等待、曝光、采集、传输和绘制;混合曝光帧不计入。相邻 rAF 夹定绘制时间,刷新间隔约 {summary.refresh.toFixed(1)} ms;面板像素响应需高速相机复核。</p>
</main>;
}