//! Camera firmware for an ESP32-S3-N16R8 with an OV2640 / OV3660 / OV5640 sensor, //! written in Rust on top of ESP-IDF. //! //! Captures JPEG frames and streams them as UDP datagrams to any number of //! clients. The wire format lives in the `protocol` crate, shared with the desktop //! client in `client/`; see that crate's documentation for why the transport is //! UDP rather than TCP. //! //! WiFi credentials come from the `WIFI_SSID` / `WIFI_PASS` environment variables //! at build time, so no secrets end up in the source. use std::sync::{ atomic::{AtomicU32, Ordering}, Arc, }; use std::time::{Duration, Instant}; use anyhow::{anyhow, Result}; use embedded_svc::wifi::{ AccessPointInfo, AuthMethod, ClientConfiguration, Configuration, PmfConfiguration, }; use esp_idf_svc::hal::peripherals::Peripherals; use esp_idf_svc::wifi::{BlockingWifi, EspWifi, WifiEvent}; use esp_idf_svc::{eventloop::EspSystemEventLoop, nvs::EspDefaultNvsPartition}; use log::{error, info, warn}; mod camera; mod link_control; mod stream_control; mod udp; use camera::{Camera, Resolution, SensorHint}; use stream_control::{Pacer, QualityController, ThermalLimiter}; use udp::UdpSender; const SSID: &str = env!("WIFI_SSID"); const PASSWORD: &str = env!("WIFI_PASS"); /// Authentication mode to assume, overriding whatever the scan reports. /// /// Usually left unset: the scan tells us the AP's real mode. Pin it with /// `WIFI_AUTH=wpa2` (or `wpa3`, `wpa2wpa3`, `wpa`, `none`) when the scan cannot /// see the AP - a hidden SSID, for instance. fn auth_override() -> Option { let name = option_env!("WIFI_AUTH")?; Some(match name { "none" => AuthMethod::None, "wep" => AuthMethod::WEP, "wpa" => AuthMethod::WPA, "wpa2" => AuthMethod::WPA2Personal, "wpawpa2" => AuthMethod::WPAWPA2Personal, "wpa3" => AuthMethod::WPA3Personal, "wpa2wpa3" => AuthMethod::WPA2WPA3Personal, "wapi" => AuthMethod::WAPIPersonal, other => panic!( "WIFI_AUTH={other} is not one of: \ none, wep, wpa, wpa2, wpawpa2, wpa3, wpa2wpa3, wapi" ), }) } /// Capture resolution, overridable at build time: `CAM_RES=vga ./scripts/build.sh` const RESOLUTION: &str = match option_env!("CAM_RES") { Some(value) => value, None => "vga", }; /// JPEG quality 0-63 (lower is better), overridable: `CAM_QUALITY=15 ./scripts/build.sh` const QUALITY: &str = match option_env!("CAM_QUALITY") { Some(value) => value, None => "12", }; /// Whether the driver inverts VSYNC, overridable so polarity can be A/B tested /// without editing code: `CAM_VSYNC_INVERT=0 ./scripts/build.sh`. /// /// A function rather than a `const` because matching on a `str` is not yet /// allowed in const context on this toolchain. /// /// `true` is esp32-camera's default and correct for most DVP sensors. `false` is /// what a board that inverts VSYNC in hardware needs. fn vsync_invert() -> bool { !matches!(option_env!("CAM_VSYNC_INVERT"), Some("0")) } fn main() -> Result<()> { // Must run once so the esp-idf-sys runtime patches link correctly. esp_idf_svc::sys::link_patches(); esp_idf_svc::log::EspLogger::initialize_default(); let resolution: Resolution = RESOLUTION.parse()?; let quality: u8 = QUALITY.parse()?; let quality_ceiling: u8 = option_env!("CAM_QUALITY_MAX").unwrap_or("0").parse()?; if quality_ceiling != 0 && !(quality..=63).contains(&quality_ceiling) { return Err(anyhow!( "CAM_QUALITY_MAX must be 0 (fixed quality) or CAM_QUALITY..63" )); } let max_fps: u32 = option_env!("CAM_MAX_FPS").unwrap_or("0").parse()?; if max_fps > 60 { return Err(anyhow!("CAM_MAX_FPS must be 0 (uncapped) or 1-60")); } let grayscale = match option_env!("CAM_GRAYSCALE").unwrap_or("0") { "0" => false, "1" => true, _ => return Err(anyhow!("CAM_GRAYSCALE must be 0 or 1")), }; let thermal_limit: f32 = option_env!("CAM_THERMAL_LIMIT_C").unwrap_or("75").parse()?; if !thermal_limit.is_finite() || (thermal_limit != 0.0 && !(40.0..=95.0).contains(&thermal_limit)) { return Err(anyhow!("CAM_THERMAL_LIMIT_C must be 0 (disabled) or 40-95")); } info!("reset reason: {}", unsafe { esp_idf_svc::sys::esp_reset_reason() }); info!("vsync invert: {}", vsync_invert()); let camera = Camera::init(SensorHint::Auto, resolution, quality, vsync_invert())?; camera.set_clock_profile(option_env!("CAM_CLOCK_PROFILE").unwrap_or("default"))?; camera.set_grayscale(grayscale)?; let exposure_ev: i32 = option_env!("CAM_EXPOSURE_EV").unwrap_or("0").parse()?; esp_idf_svc::sys::esp!(unsafe { esp_idf_svc::sys::camera::cam_wrapper_set_exposure_bias(exposure_ev) })?; if let Some(level) = option_env!("CAM_AE_LEVEL") { if exposure_ev != 0 { return Err(anyhow!( "Use CAM_AE_LEVEL or a nonzero CAM_EXPOSURE_EV, not both" )); } camera.set_ae_level(level.parse()?)?; } let horizontal_total: u32 = option_env!("CAM_HTS").unwrap_or("0").parse()?; esp_idf_svc::sys::esp!(unsafe { esp_idf_svc::sys::camera::cam_wrapper_set_horizontal_timing(horizontal_total) })?; let shutter_lines: u32 = option_env!("CAM_SHUTTER_LINES").unwrap_or("0").parse()?; let exposure_limit: u32 = option_env!("CAM_EXPOSURE_MAX_LINES") .unwrap_or("0") .parse()?; if shutter_lines != 0 && exposure_limit != 0 { return Err(anyhow!( "Use CAM_SHUTTER_LINES or CAM_EXPOSURE_MAX_LINES, not both" )); } esp_idf_svc::sys::esp!(unsafe { esp_idf_svc::sys::camera::cam_wrapper_set_shutter_lines(shutter_lines) })?; esp_idf_svc::sys::esp!(unsafe { esp_idf_svc::sys::camera::cam_wrapper_set_exposure_limit(exposure_limit) })?; camera.set_gain_ceiling(option_env!("CAM_GAIN_CEILING").unwrap_or("0").parse()?)?; info!("stream profile: target {max_fps} fps, grayscale {grayscale}, quality {quality}, thermal limit {thermal_limit:.1} C"); info!( "configured capture probe: {:.2} fps", camera.probe_frame_rate() ); let temperature_available = unsafe { esp_idf_svc::sys::camera::cam_wrapper_temperature_init() }; if temperature_available != 0 { warn!("chip temperature unavailable (esp_err 0x{temperature_available:x}); fixed frame cap remains active"); } let peripherals = Peripherals::take()?; let sys_loop = EspSystemEventLoop::take()?; let nvs = EspDefaultNvsPartition::take()?; let network_epoch = Arc::new(AtomicU32::new(0)); let event_epoch = network_epoch.clone(); let _wifi_events = sys_loop.subscribe::(move |event| { if matches!(event, WifiEvent::StaDisconnected(_)) { event_epoch.fetch_add(1, Ordering::Relaxed); } })?; let mut wifi = BlockingWifi::wrap( EspWifi::new(peripherals.modem, sys_loop.clone(), Some(nvs))?, sys_loop, )?; // `connect` starts the driver, then scans and associates, so that an SSID on // a 5 GHz network (which this radio cannot see at all) is reported as such // instead of a bare association timeout. if let Err(err) = connect(&mut wifi) { error!("could not join \"{SSID}\": {err}"); error!("check that the SSID is a 2.4 GHz network and the password is right"); return Err(err); } let ip = wifi.wifi().sta_netif().get_ip_info()?.ip; configure_radio()?; // The board identifies itself by an id derived from its MAC, not its IP: DHCP // hands out a different address across reboots, and a client that saved its // binding to one would have to be reconfigured every time. let sender_id = udp::local_sender_id(); let mut sender = UdpSender::new(sender_id)?; info!( "RMIC::2026 firmware {} (build {}) on {ip}, wire protocol {}", udp::FIRMWARE_VERSION, env!("RMIC_BUILD"), protocol::PROTOCOL_VERSION ); info!("clients will find it by broadcast; no configuration needed on this side"); let mut last_report = Instant::now(); let mut last_stats = sender.stats(); let mut pacer = Pacer::new(max_fps); let mut thermal = ThermalLimiter::new(max_fps, thermal_limit); let mut current_quality = quality; let mut quality_control = QualityController::new( quality, if quality_ceiling == 0 { quality } else { quality_ceiling }, ); let loop_start = Instant::now(); let mut recovery = link_control::Recovery::new(0); let mut seen_epoch = network_epoch.load(Ordering::Relaxed); let mut link_up = true; let mut active_ip = ip; let mut next_link_check = 0u64; let mut outage_started = 0u64; let test_drop_ms = option_env!("WIFI_TEST_DROP_AFTER_S") .unwrap_or("0") .parse::()? .saturating_mul(1000); let mut test_injected = false; let mut test_hold_until = 0u64; loop { let now_ms = loop_start.elapsed().as_millis() as u64; if test_drop_ms > 0 && !test_injected && now_ms >= test_drop_ms { test_injected = true; test_hold_until = now_ms + 5000; warn!("Wi-Fi recovery self-test: intentional disconnect for 5000 ms"); wifi.wifi_mut().disconnect().ok(); next_link_check = 0; } if now_ms >= next_link_check { next_link_check = now_ms + 100; let epoch = network_epoch.load(Ordering::Relaxed); let associated = wifi.wifi().is_connected().unwrap_or(false); let lease = wifi .wifi() .sta_netif() .get_ip_info() .ok() .map(|v| v.ip) .filter(|ip| !ip.is_unspecified()); let ready = associated && wifi.wifi().is_up().unwrap_or(false) && lease.is_some() && now_ms >= test_hold_until; let address_changed = ready && lease.is_some_and(|address| address != active_ip); if link_up && (!ready || epoch != seen_epoch || address_changed) { link_up = false; outage_started = now_ms; recovery = link_control::Recovery::new(now_ms); sender.network_lost(); warn!("Wi-Fi link unavailable or changed; UDP suspended, waiting for association and DHCP"); } else if epoch != seen_epoch { recovery.disconnected(now_ms); } seen_epoch = epoch; if !link_up && ready { match sender.restore_network() { Ok(()) => { active_ip = lease.unwrap(); if let Err(err) = configure_radio() { warn!("radio profile restore: {err}"); } // Discard any queued image from before recovery. for _ in 0..2 { if let Ok(frame) = camera.next_frame() { drop(frame); } } link_up = true; last_report = Instant::now(); last_stats = sender.stats(); info!("Wi-Fi recovery complete after {} ms; IP {active_ip}; UDP recreated, camera state preserved",now_ms-outage_started); } Err(err) => warn!("UDP restore failed, will retry: {err}"), } } else if !link_up && now_ms >= test_hold_until { match recovery.poll(now_ms, associated) { Some(link_control::Action::Connect) => { info!("Wi-Fi reconnect attempt"); if let Err(err) = wifi.wifi_mut().connect() { warn!("Wi-Fi reconnect rejected: {err}"); recovery.disconnected(now_ms); } } Some(link_control::Action::AbortAttempt) => { warn!("Wi-Fi association/DHCP deadline elapsed; cancelling attempt before backoff"); wifi.wifi_mut().disconnect().ok(); } None => {} } } } if !link_up { std::thread::sleep(Duration::from_millis(50)); continue; } sender.housekeeping(); let now_us = unsafe { esp_idf_svc::sys::esp_timer_get_time() } as u64; let wait_us = pacer.delay_us(now_us); if wait_us > 0 { std::thread::sleep(Duration::from_micros(wait_us)); } pacer.started(unsafe { esp_idf_svc::sys::esp_timer_get_time() } as u64); udp::pump(&camera, &mut sender)?; let now = Instant::now(); if now.duration_since(last_report) >= Duration::from_secs(5) { // Deltas rather than totals: the totals alone cannot distinguish "the // sensor is slow" from "the sender is not keeping up", which is the // distinction that matters when the frame rate is wrong. let s = sender.stats(); let elapsed = now.duration_since(last_report).as_secs_f32(); let attempted = s.frames_attempted - last_stats.frames_attempted; let dropped = s.peer_frames_dropped - last_stats.peer_frames_dropped; let send_ms = (s.send_time_us - last_stats.send_time_us) as f32 / attempted.max(1) as f32 / 1000.0; if let Some(next) = quality_control.observe( u64::from(attempted) * sender.peer_count() as u64, dropped, send_ms, ) { match camera.set_quality(next) { Ok(()) => { info!("JPEG load control: quality {current_quality} -> {next}, {dropped} peer drops / {attempted} attempts, {send_ms:.2} ms/send"); current_quality = next; } Err(err) => { warn!("JPEG load control disabled after sensor error: {err}"); quality_control = QualityController::new(current_quality, current_quality); } } } let mut ap: esp_idf_svc::sys::wifi_ap_record_t = unsafe { std::mem::zeroed() }; let ap_ok = unsafe { esp_idf_svc::sys::esp_wifi_sta_get_ap_info(&mut ap) } == 0; let mut tx_power = 0; let tx_power_ok = unsafe { esp_idf_svc::sys::esp_wifi_get_max_tx_power(&mut tx_power) } == 0; let mut celsius = f32::NAN; let temp_ok = temperature_available == 0 && unsafe { esp_idf_svc::sys::camera::cam_wrapper_temperature_read(&mut celsius) } == 0; let previous_fps = thermal.fps(); let target_fps = thermal.observe(if temp_ok { Some(celsius) } else { None }); if previous_fps != target_fps { pacer.set_fps( target_fps, unsafe { esp_idf_svc::sys::esp_timer_get_time() } as u64, ); info!("thermal load limit changed: {previous_fps} -> {target_fps} fps at {celsius:.1} C"); } info!( "{:.1} fps submitted, {:.1} fps attempted, {:.1} datagrams/s, {:.1} kB/s, \ {} send failures, {} retries, errno {}, {} peer frames dropped, \ {:.2} ms/send, {} clients subscribed, {} frames skipped (no client), \ internal heap {} free / {} largest bytes, RSSI {} dBm channel {}, \ chip temp {:.1} C, target {} fps, thermal_limited {}, capture errors {}, exposure {} lines, jpeg quality {}, gain {:.2}x, \ capture queue skipped {}, stale frames dropped {}, deadline peer drops {}, max submitted capture age {:.2} ms, max send hold {:.2} ms, max TX power {:.2} dBm", (s.frames_sent - last_stats.frames_sent) as f32 / elapsed, (s.frames_attempted - last_stats.frames_attempted) as f32 / elapsed, (s.fragments_sent - last_stats.fragments_sent) as f32 / elapsed, (s.bytes_sent - last_stats.bytes_sent) as f32 / elapsed / 1024.0, s.errors - last_stats.errors, s.retries - last_stats.retries, s.last_errno, s.peer_frames_dropped - last_stats.peer_frames_dropped, (s.send_time_us - last_stats.send_time_us) as f32 / (s.frames_attempted - last_stats.frames_attempted).max(1) as f32 / 1000.0, sender.peer_count(), s.skipped_no_peer - last_stats.skipped_no_peer, unsafe { esp_idf_svc::sys::heap_caps_get_free_size(esp_idf_svc::sys::MALLOC_CAP_INTERNAL) }, unsafe { esp_idf_svc::sys::heap_caps_get_largest_free_block( esp_idf_svc::sys::MALLOC_CAP_INTERNAL, ) }, if ap_ok { ap.rssi } else { 0 }, if ap_ok { ap.primary } else { 0 }, if temp_ok { celsius } else { f32::NAN }, target_fps, thermal.limited(), s.capture_errors - last_stats.capture_errors, unsafe { esp_idf_svc::sys::camera::cam_wrapper_exposure_lines() }, current_quality, unsafe { esp_idf_svc::sys::camera::cam_wrapper_gain_raw() } as f32 / 16.0, s.capture_queue_skipped - last_stats.capture_queue_skipped, s.stale_frames_dropped - last_stats.stale_frames_dropped, s.deadline_peer_drops - last_stats.deadline_peer_drops, s.max_submitted_age_us as f32 / 1000.0, s.max_send_hold_us as f32 / 1000.0, if tx_power_ok { tx_power as f32 / 4.0 } else { f32::NAN }, ); last_stats = s; last_report = now; } } } fn configure_radio() -> Result<()> { // Modem power save trades latency for current, and for a video stream that // trade is backwards: with PS on, an idle beacon interval costs hundreds of // milliseconds, which shows up as stutter and a ping of 200-300 ms on an // otherwise good link. The board is mains-powered anyway. // // This one binds as a bare `esp_err_t` rather than a `Result`. let ps_rc = unsafe { esp_idf_svc::sys::esp_wifi_set_ps(esp_idf_svc::sys::wifi_ps_type_t_WIFI_PS_NONE) }; if ps_rc == 0 { info!("WiFi power save disabled"); } else { warn!("could not disable WiFi power save (esp_err 0x{ps_rc:x})"); } // Apply the selected power cap both at startup and after reconnecting. // Congestion/thermal controllers must not lower it: they adjust JPEG // quality or frame delivery instead. The SDK quantizes this setting. if let Some(value) = option_env!("WIFI_TX_POWER_DBM") { let dbm: u8 = value .parse() .map_err(|_| anyhow!("WIFI_TX_POWER_DBM must be an integer from 2 to 20"))?; if !(2..=20).contains(&dbm) { return Err(anyhow!("WIFI_TX_POWER_DBM must be an integer from 2 to 20")); } esp_idf_svc::sys::esp!(unsafe { esp_idf_svc::sys::esp_wifi_set_max_tx_power((dbm * 4) as i8) })?; info!("WiFi requested max TX power: {dbm} dBm"); } let mut ap: esp_idf_svc::sys::wifi_ap_record_t = unsafe { std::mem::zeroed() }; let mut phy = 0; let mut tx_power = 0; unsafe { esp_idf_svc::sys::esp_wifi_sta_get_ap_info(&mut ap); esp_idf_svc::sys::esp_wifi_sta_get_negotiated_phymode(&mut phy); } esp_idf_svc::sys::esp!(unsafe { esp_idf_svc::sys::esp_wifi_get_max_tx_power(&mut tx_power) })?; info!( "WiFi link: negotiated PHY {phy}, pairwise cipher {}, max TX power {:.2} dBm", ap.pairwise_cipher, tx_power as f32 / 4.0 ); Ok(()) } /// Joins `SSID`, scanning first and retrying the plausible authentication modes. /// /// The scan is not decoration. This radio is 2.4 GHz only, so a 5 GHz SSID never /// shows up at all and a bare "association timed out" tells you nothing. /// /// The retries are because the AP's advertised mode is not the same thing as the /// mode it will accept. Routers commonly run WPA/WPA2 mixed, WPA2/WPA3 mixed, or /// WPA3 with PMF mandatory, and a client pinned to the wrong one fails the 4-way /// handshake *after* it has already associated - the failure looks identical to /// a wrong password, so it is worth walking the list instead of guessing. fn connect(wifi: &mut BlockingWifi>) -> Result<()> { let pinned_auth = auth_override(); // A configuration has to exist before start(), which is what decides // station vs AP mode; the real one is picked after the scan. wifi.set_configuration(&client_config( pinned_auth.unwrap_or_else(default_auth_method), )?)?; wifi.start()?; info!("wifi started"); info!("scanning for 2.4 GHz networks..."); let advertised = match wifi.wifi_mut().scan() { Ok(aps) => { report_scan(&aps); advertised_auth(&aps) } Err(err) => { warn!("scan failed: {err}"); None } }; for auth in candidates(advertised, pinned_auth) { info!("trying {auth:?}..."); wifi.set_configuration(&client_config(auth)?)?; // Driver-level (non-blocking) connect: `BlockingWifi::connect()` waits // out the driver's own ~20 s association timeout, which would make // walking the list of modes take minutes. wifi.wifi_mut().connect()?; // Bounded, so one bad mode costs a few seconds instead of the timeout. let associated = wifi .wifi_wait_while( || wifi.wifi().is_connected().map(|c| !c), Some(Duration::from_secs(5)), ) .is_ok(); let got_ip = associated && wifi .ip_wait_while( || wifi.wifi().is_up().map(|up| !up), Some(Duration::from_secs(4)), ) .is_ok(); if got_ip { let ip_info = wifi.wifi().sta_netif().get_ip_info()?; if !ip_info.ip.is_unspecified() { info!("wifi connected with {auth:?}: {ip_info:?}"); return Ok(()); } } warn!( "{auth:?} did not complete the handshake ({})", if associated { "associated but no DHCP address" } else { "no association" } ); wifi.wifi_mut().disconnect().ok(); } Err(anyhow!( "no authentication mode joined \"{SSID}\" (tried {:?}); \ association succeeded but the 4-way handshake did not, \ which almost always means the password is wrong", candidates(advertised, pinned_auth) )) } /// Authentication modes to try, most likely first. /// /// A mode pinned with `WIFI_AUTH` is the only one tried - if it does not work, /// trying others would just be guessing at a value the builder already stated. fn candidates(advertised: Option, pinned: Option) -> Vec { if let Some(auth) = pinned { return vec![auth]; } // "WPA" proper is a pre-2004 mode no current router speaks, and "None" is // only meaningful without a password, so neither is worth a 10 s attempt. let mut modes = vec![advertised.unwrap_or_else(default_auth_method)]; modes.extend([ AuthMethod::WPA2Personal, AuthMethod::WPA2WPA3Personal, AuthMethod::WPA3Personal, ]); if PASSWORD.is_empty() { modes.push(AuthMethod::None); } let mut seen = Vec::new(); modes.retain(|m| { let fresh = !seen.contains(m); seen.push(*m); fresh }); modes } fn client_config(auth_method: AuthMethod) -> Result { Ok(Configuration::Client(ClientConfiguration { ssid: SSID .try_into() .map_err(|_| anyhow!("WIFI_SSID too long (max 32 bytes)"))?, bssid: None, auth_method, password: PASSWORD .try_into() .map_err(|_| anyhow!("WIFI_PASS too long (max 64 bytes)"))?, channel: None, // Advertising PMF keeps routers that offer WPA2+WPA3 mixed mode from // dropping the handshake, while `required: false` still lets a WPA2-only // AP through. Leaving this at NotCapable fails on the former. pmf_cfg: PmfConfiguration::Capable { required: false }, ..Default::default() })) } fn default_auth_method() -> AuthMethod { if PASSWORD.is_empty() { AuthMethod::None } else { AuthMethod::WPA2Personal } } /// What `SSID` advertises, if the scan saw it. /// /// `None` means the scan did not find this network at all, which on its own is /// the most useful diagnostic there is: on a 2.4 GHz-only radio it almost always /// means the SSID belongs to a 5 GHz network, or the AP is on a channel this /// region's regulatory domain forbids. fn advertised_auth(aps: &[AccessPointInfo]) -> Option { aps.iter() .find(|ap| ap.ssid.as_str() == SSID) .and_then(|ap| ap.auth_method) } /// Logs the visible networks, marking the one we are trying to join. fn report_scan(aps: &[AccessPointInfo]) { if aps.is_empty() { warn!("scan found no networks at all"); return; } for ap in aps { info!( " {:<32} ch {:>2} {:>4} dBm {:?}{}", ap.ssid, ap.channel, ap.signal_strength, ap.auth_method, if ap.ssid.as_str() == SSID { " <- target" } else { "" } ); } if advertised_auth(aps).is_none() { error!("\"{SSID}\" was not in the scan results"); error!("this radio is 2.4 GHz only, so the AP is likely 5 GHz or on a forbidden channel"); } }