diff --git a/Cargo.lock b/Cargo.lock index 5c508a7e1..8722db646 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3888,8 +3888,8 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hwcodec" -version = "0.7.1" -source = "git+https://github.com/rustdesk-org/hwcodec#778df1f99597722473b29443bac22ae6c23946fe" +version = "0.8.0" +source = "git+https://github.com/rustdesk-org/hwcodec#febec27e1b40bb9fbbae02e9c59b71ac824b9fe1" dependencies = [ "bindgen 0.59.2", "cc", diff --git a/libs/scrap/src/common/hwcodec.rs b/libs/scrap/src/common/hwcodec.rs index 9cf3367ba..806fd10fa 100644 --- a/libs/scrap/src/common/hwcodec.rs +++ b/libs/scrap/src/common/hwcodec.rs @@ -25,6 +25,9 @@ use hwcodec::{ }, }; +#[cfg(all(test, windows, feature = "vram"))] +mod tests; + const DEFAULT_PIXFMT: AVPixelFormat = AVPixelFormat::AV_PIX_FMT_NV12; pub const DEFAULT_FPS: i32 = 30; const DEFAULT_GOP: i32 = i32::MAX; @@ -236,7 +239,7 @@ impl HwRamEncoder { data.append(v); Ok(data) } - Err(_) => Ok(Vec::::new()), + Err(code) => Err(anyhow!("RAM encode failed: {}", code)), } } diff --git a/libs/scrap/src/common/hwcodec/tests.rs b/libs/scrap/src/common/hwcodec/tests.rs new file mode 100644 index 000000000..4bc9b5033 --- /dev/null +++ b/libs/scrap/src/common/hwcodec/tests.rs @@ -0,0 +1,130 @@ +use super::{CodecInfo, HwCodecConfig, CONFIG, CONFIG_SET_BY_IPC}; +use crate::{ + codec::enable_vram_option, + vram::{VRamDecoder, VRamEncoder}, + AdapterDevice, CodecFormat, +}; +use base::config::keys::{OPTION_ALLOW_D3D_RENDER, OPTION_ENABLE_HWCODEC}; +use hbb_common::config::{OVERWRITE_LOCAL_SETTINGS, OVERWRITE_SETTINGS}; +use hwcodec::{ + common::{DataFormat, Driver}, + vram::{DecodeContext, FeatureContext}, +}; +use std::{collections::HashMap, mem}; + +struct RestoreConfig { + config: Option, + set_by_ipc: bool, + options: HashMap, + local_options: HashMap, +} + +impl Drop for RestoreConfig { + fn drop(&mut self) { + *CONFIG.lock().unwrap() = self.config.take(); + *CONFIG_SET_BY_IPC.lock().unwrap() = self.set_by_ipc; + *OVERWRITE_SETTINGS.write().unwrap() = mem::take(&mut self.options); + *OVERWRITE_LOCAL_SETTINGS.write().unwrap() = mem::take(&mut self.local_options); + } +} + +fn add_backend(config: &mut HwCodecConfig, driver: Driver, vendor: Driver, format: DataFormat) { + let luid = match vendor { + Driver::NV => 1, + Driver::MFX => 2, + Driver::AMF => 3, + _ => unreachable!(), + }; + config.vram_encode.push(FeatureContext { + driver: driver.clone(), + vendor: vendor.clone(), + luid, + data_format: format, + }); + config.vram_decode.push(DecodeContext { + device: None, + driver, + vendor, + luid, + data_format: format, + }); +} + +#[test] +fn cached_sdk_entries_are_ignored_before_check_completes() { + let _restore = RestoreConfig { + config: CONFIG.lock().unwrap().take(), + set_by_ipc: mem::replace(&mut *CONFIG_SET_BY_IPC.lock().unwrap(), false), + options: mem::take(&mut *OVERWRITE_SETTINGS.write().unwrap()), + local_options: mem::take(&mut *OVERWRITE_LOCAL_SETTINGS.write().unwrap()), + }; + OVERWRITE_SETTINGS + .write() + .unwrap() + .insert(OPTION_ENABLE_HWCODEC.into(), "Y".into()); + OVERWRITE_LOCAL_SETTINGS + .write() + .unwrap() + .insert(OPTION_ALLOW_D3D_RENDER.into(), "Y".into()); + assert!(enable_vram_option(false)); + + let formats = [ + (CodecFormat::H264, DataFormat::H264), + (CodecFormat::H265, DataFormat::H265), + ]; + let mut cached = HwCodecConfig::default(); + for (_, format) in formats { + // A RAM fallback keeps this selection test independent of attached displays. + cached.ram_encode.push(CodecInfo { + format, + ..Default::default() + }); + for vendor in [Driver::NV, Driver::MFX, Driver::AMF] { + add_backend(&mut cached, vendor.clone(), vendor, format); + } + } + *CONFIG.lock().unwrap() = Some(cached.clone()); + assert!(!HwCodecConfig::already_set()); + for (format, _) in formats { + assert!(VRamEncoder::available(format).is_empty()); + for luid in 1..=3 { + assert!(VRamDecoder::try_get(format, Some(luid)).is_none()); + } + } + assert_eq!( + VRamDecoder::possible_available_without_check(), + (false, false) + ); + + for (index, &(format, data_format)) in formats.iter().enumerate() { + for vendor in [Driver::NV, Driver::MFX, Driver::AMF] { + add_backend(&mut cached, Driver::FFMPEG, vendor, data_format); + } + *CONFIG.lock().unwrap() = Some(cached.clone()); + let encoders = VRamEncoder::available(format); + assert_eq!(encoders.len(), 3); + for encoder in encoders { + assert_eq!(encoder.driver, Driver::FFMPEG); + let device = AdapterDevice { + luid: encoder.luid, + ..Default::default() + }; + assert_eq!(VRamEncoder::try_get(&device, format), Some(encoder.clone())); + let decoder = VRamDecoder::try_get(format, Some(encoder.luid)).unwrap(); + assert_eq!(decoder.driver, Driver::FFMPEG); + assert_eq!(decoder.vendor, encoder.vendor); + assert_eq!(decoder.data_format, data_format); + } + assert!(VRamDecoder::available(format, Some(99)).is_empty()); + assert!(VRamDecoder::available(format, None).is_empty()); + assert_eq!( + VRamDecoder::possible_available_without_check(), + (true, index == 1) + ); + } + assert!(!HwCodecConfig::already_set()); + assert_eq!( + HwCodecConfig::get().vram_encode.len(), + cached.vram_encode.len() + ); +} diff --git a/libs/scrap/src/common/mod.rs b/libs/scrap/src/common/mod.rs index 8c14db13b..167002086 100644 --- a/libs/scrap/src/common/mod.rs +++ b/libs/scrap/src/common/mod.rs @@ -209,6 +209,8 @@ impl Frame<'_> { pub enum EncodeInput<'a> { YUV(&'a [u8]), Texture((*mut c_void, usize)), + #[cfg(all(windows, feature = "vram"))] + Repeat, } impl<'a> EncodeInput<'a> { diff --git a/libs/scrap/src/common/vpxcodec.rs b/libs/scrap/src/common/vpxcodec.rs index 0478f7fa3..d7413cdfe 100644 --- a/libs/scrap/src/common/vpxcodec.rs +++ b/libs/scrap/src/common/vpxcodec.rs @@ -20,6 +20,17 @@ use std::{ptr, slice}; generate_call_macro!(call_vpx, false); generate_call_ptr_macro!(call_vpx_ptr); +#[derive(Debug)] +pub struct VpxNoOutput; + +impl std::fmt::Display for VpxNoOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("no valid frame") + } +} + +impl std::error::Error for VpxNoOutput {} + #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] pub enum VpxVideoCodecId { VP8, @@ -187,7 +198,7 @@ impl EncoderApi for VpxEncoder { if frames.len() > 0 { Ok(VpxEncoder::create_video_frame(self.id, frames)) } else { - Err(anyhow!("no valid frame")) + Err(VpxNoOutput.into()) } } diff --git a/libs/scrap/src/common/vram.rs b/libs/scrap/src/common/vram.rs index 16e21cd7a..be3867345 100644 --- a/libs/scrap/src/common/vram.rs +++ b/libs/scrap/src/common/vram.rs @@ -15,8 +15,9 @@ use hbb_common::{ bytes::Bytes, log, ResultType, }; +pub use hwcodec::common::Driver; use hwcodec::{ - common::{DataFormat, Driver, MAX_GOP}, + common::{DataFormat, MAX_GOP}, vram::{ decode::{self, DecodeFrame, Decoder}, encode::{self, EncodeFrame, Encoder}, @@ -24,6 +25,9 @@ use hwcodec::{ }, }; +#[cfg(test)] +mod repeat_tests; + // https://www.reddit.com/r/buildapc/comments/d2m4ny/two_graphics_cards_two_monitors/ // https://www.reddit.com/r/techsupport/comments/t2v9u6/dual_monitor_setup_with_dual_gpu/ // https://cybersided.com/two-monitors-two-gpus/ @@ -48,8 +52,6 @@ pub struct VRamEncoder { pub format: DataFormat, ctx: EncodeContext, bitrate: u32, - last_frame_len: usize, - same_bad_len_counter: usize, } impl EncoderApi for VRamEncoder { @@ -83,8 +85,6 @@ impl EncoderApi for VRamEncoder { ctx, format: config.feature.data_format, bitrate, - last_frame_len: 0, - same_bad_len_counter: 0, }), Err(_) => Err(anyhow!(format!("Failed to create encoder"))), } @@ -98,7 +98,15 @@ impl EncoderApi for VRamEncoder { frame: EncodeInput, ms: i64, ) -> ResultType { - let (texture, rotation) = frame.texture()?; + #[cfg(all(windows, feature = "vram"))] + let repeated = matches!(&frame, EncodeInput::Repeat); + #[cfg(not(all(windows, feature = "vram")))] + let repeated = false; + let (texture, rotation) = if repeated { + (std::ptr::null_mut(), 0) + } else { + frame.texture()? + }; if rotation != 0 { // to-do: support rotation // Both the encoder and display(w,h) information need to be changed. @@ -106,10 +114,12 @@ impl EncoderApi for VRamEncoder { } let mut vf = VideoFrame::new(); let mut frames = Vec::new(); - for frame in self - .encode(texture, ms) - .with_context(|| "Failed to encode")? - { + let result = if repeated { + self.encode_repeat(ms) + } else { + self.encode(texture, ms) + }; + for frame in result.with_context(|| "Failed to encode")? { frames.push(EncodedVideoFrame { data: Bytes::from(frame.data), pts: frame.pts, @@ -117,29 +127,10 @@ impl EncoderApi for VRamEncoder { ..Default::default() }); } + if repeated && frames.is_empty() { + return Ok(vf); + } if frames.len() > 0 { - // This kind of problem is occurred after a period of time when using the AMD SDK, - // the encoding length is fixed at about 40, and the picture is still - const MIN_BAD_LEN: usize = 100; - const MAX_BAD_COUNTER: usize = 30; - let this_frame_len = frames[0].data.len(); - if self.ctx.f.driver == Driver::AMF - && this_frame_len < MIN_BAD_LEN - && this_frame_len == self.last_frame_len - { - self.same_bad_len_counter += 1; - if self.same_bad_len_counter >= MAX_BAD_COUNTER { - log::info!( - "{} times encoding len is {}, switch", - self.same_bad_len_counter, - self.last_frame_len - ); - bail!(crate::codec::ENCODE_NEED_SWITCH); - } - } else { - self.same_bad_len_counter = 0; - } - self.last_frame_len = this_frame_len; let frames = EncodedVideoFrames { frames: frames.into(), ..Default::default() @@ -244,6 +235,7 @@ impl VRamEncoder { let v: Vec<_> = crate::hwcodec::HwCodecConfig::get() .vram_encode .drain(..) + .filter(|c| c.driver == Driver::FFMPEG) .filter(|c| c.data_format == data_format) .collect(); if crate::hwcodec::HwRamEncoder::try_get(format).is_some() { @@ -274,14 +266,21 @@ impl VRamEncoder { } } - pub fn encode(&mut self, texture: *mut c_void, ms: i64) -> ResultType> { + pub fn encode( + &mut self, + texture: *mut c_void, + ms: i64, + ) -> ResultType> { match self.encoder.encode(texture, ms) { - Ok(v) => { - let mut data = Vec::::new(); - data.append(v); - Ok(data) - } - Err(_) => Ok(Vec::::new()), + Ok(frames) => Ok(frames.drain(..)), + Err(code) => Err(anyhow!("VRAM encode failed: {}", code)), + } + } + + fn encode_repeat(&mut self, ms: i64) -> ResultType> { + match self.encoder.encode_repeat(ms) { + Ok(frames) => Ok(frames.drain(..)), + Err(code) => Err(anyhow!("VRAM repeat encode failed: {}", code)), } } @@ -340,6 +339,7 @@ impl VRamDecoder { crate::hwcodec::HwCodecConfig::get() .vram_decode .drain(..) + .filter(|c| c.driver == Driver::FFMPEG) .filter(|c| c.data_format == data_format && c.luid == luid && luid != 0) .collect() } @@ -350,8 +350,10 @@ impl VRamDecoder { } let v = crate::hwcodec::HwCodecConfig::get().vram_decode; ( - v.iter().any(|d| d.data_format == DataFormat::H264), - v.iter().any(|d| d.data_format == DataFormat::H265), + v.iter() + .any(|d| d.driver == Driver::FFMPEG && d.data_format == DataFormat::H264), + v.iter() + .any(|d| d.driver == Driver::FFMPEG && d.data_format == DataFormat::H265), ) } diff --git a/libs/scrap/src/common/vram/repeat_tests.rs b/libs/scrap/src/common/vram/repeat_tests.rs new file mode 100644 index 000000000..e09e9878e --- /dev/null +++ b/libs/scrap/src/common/vram/repeat_tests.rs @@ -0,0 +1,244 @@ +use super::VRamEncoder; +use crate::{codec::EncoderApi, EncodeInput}; +use base::message_proto::{video_frame, VideoFrame}; +use hwcodec::{ + common::{DataFormat, Driver, MAX_GOP}, + ffmpeg::AVHWDeviceType, + ffmpeg_ram::decode::{DecodeContext, Decoder}, + vram::{encode::Encoder, DynamicContext, EncodeContext, FeatureContext}, +}; +use std::{mem, ptr}; +use winapi::{ + shared::{ + dxgi::{CreateDXGIFactory1, IDXGIFactory1}, + dxgiformat::DXGI_FORMAT_B8G8R8A8_UNORM, + dxgitype::DXGI_SAMPLE_DESC, + winerror::DXGI_ERROR_NOT_FOUND, + }, + um::{d3d11::*, d3dcommon::D3D_DRIVER_TYPE_UNKNOWN, unknwnbase::IUnknown}, + Interface, +}; + +struct Com(*mut T); + +impl Drop for Com { + fn drop(&mut self) { + if !self.0.is_null() { + unsafe { (*(self.0 as *mut IUnknown)).Release() }; + } + } +} + +unsafe fn texture( + device: *mut ID3D11Device, + width: u32, + height: u32, + shade: u8, +) -> Com { + let pixels = pixels(width, height, shade); + let desc = D3D11_TEXTURE2D_DESC { + Width: width, + Height: height, + MipLevels: 1, + ArraySize: 1, + Format: DXGI_FORMAT_B8G8R8A8_UNORM, + SampleDesc: DXGI_SAMPLE_DESC { + Count: 1, + Quality: 0, + }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: D3D11_BIND_SHADER_RESOURCE | D3D11_BIND_RENDER_TARGET, + CPUAccessFlags: 0, + MiscFlags: D3D11_RESOURCE_MISC_SHARED, + }; + let data = D3D11_SUBRESOURCE_DATA { + pSysMem: pixels.as_ptr() as _, + SysMemPitch: width * 4, + SysMemSlicePitch: 0, + }; + let mut raw = ptr::null_mut(); + assert_eq!((*device).CreateTexture2D(&desc, &data, &mut raw), 0); + Com(raw) +} + +fn pixels(width: u32, height: u32, shade: u8) -> Vec { + [shade, shade, shade, 255].repeat((width * height) as usize) +} + +fn check_picture( + decoder: &mut Decoder, + frame: VideoFrame, + width: u32, + height: u32, + bright: bool, + pts: i64, +) { + let frames = match frame.union.unwrap() { + video_frame::Union::H264s(frames) | video_frame::Union::H265s(frames) => frames, + _ => panic!("unexpected codec"), + }; + let mut decoded = 0; + for packet in frames.frames { + assert_eq!(packet.pts, pts); + for picture in decoder.decode(&packet.data).unwrap() { + assert_eq!((picture.width, picture.height), (width as _, height as _)); + let center = (height as usize / 2) * picture.linesize[0] as usize + width as usize / 2; + let y = picture.data[0][center]; + assert!( + if bright { y > 160 } else { y < 90 }, + "stale or overwritten pixels: expected bright={}, got Y={}", + bright, + y + ); + decoded += 1; + } + } + assert!(decoded > 0, "encoder output did not decode to a picture"); +} + +#[test] +#[ignore = "requires hardware H.264/H.265 encoders; run explicitly on GPU hosts"] +fn hardware_repeats_own_pixels_and_reset_on_recreation() { + unsafe { + let mut raw_factory = ptr::null_mut(); + assert_eq!( + CreateDXGIFactory1(&IDXGIFactory1::uuidof(), &mut raw_factory), + 0 + ); + let factory = Com(raw_factory as *mut IDXGIFactory1); + let mut tested = 0; + for index in 0.. { + let mut raw_adapter = ptr::null_mut(); + let result = (*factory.0).EnumAdapters1(index, &mut raw_adapter); + if result == DXGI_ERROR_NOT_FOUND { + break; + } + assert_eq!(result, 0); + let adapter = Com(raw_adapter); + let mut desc = mem::zeroed(); + assert_eq!((*adapter.0).GetDesc1(&mut desc), 0); + let vendor = match desc.VendorId { + 0x10de => Driver::NV, + 0x1002 => Driver::AMF, + 0x8086 => Driver::MFX, + _ => continue, + }; + let luid = ((desc.AdapterLuid.HighPart as i64) << 32) | desc.AdapterLuid.LowPart as i64; + let mut raw_device = ptr::null_mut(); + let mut raw_context = ptr::null_mut(); + assert_eq!( + D3D11CreateDevice( + adapter.0 as _, + D3D_DRIVER_TYPE_UNKNOWN, + ptr::null_mut(), + D3D11_CREATE_DEVICE_BGRA_SUPPORT | D3D11_CREATE_DEVICE_VIDEO_SUPPORT, + ptr::null(), + 0, + D3D11_SDK_VERSION, + &mut raw_device, + ptr::null_mut(), + &mut raw_context, + ), + 0 + ); + let device = Com(raw_device); + let context = Com(raw_context); + for driver in [Driver::FFMPEG, vendor.clone()] { + for format in [DataFormat::H264, DataFormat::H265] { + for (width, height) in [(320, 240), (640, 360)] { + let ctx = EncodeContext { + f: FeatureContext { + driver: driver.clone(), + vendor: vendor.clone(), + luid, + data_format: format, + }, + d: DynamicContext { + device: Some(device.0 as _), + width: width as _, + height: height as _, + kbitrate: 1000, + framerate: 30, + gop: MAX_GOP as _, + }, + }; + let native = match Encoder::new(ctx.clone()) { + Ok(encoder) => encoder, + Err(()) => { + eprintln!( + "SKIP unavailable {:?} {:?} {:?} adapter={luid:#x}", + vendor, driver, format + ); + continue; + } + }; + let mut encoder = VRamEncoder { + encoder: native, + format, + ctx, + bitrate: 1000, + }; + assert!(encoder.encode_to_message(EncodeInput::Repeat, 0).is_err()); + assert!(encoder + .encode_to_message(EncodeInput::Texture((ptr::null_mut(), 0)), 0) + .is_err()); + let mut decoder = Decoder::new(DecodeContext { + name: if format == DataFormat::H264 { + "h264" + } else { + "hevc" + } + .into(), + device_type: AVHWDeviceType::AV_HWDEVICE_TYPE_NONE, + thread_count: 1, + }) + .unwrap(); + let source = texture(device.0, width, height, 32); + let frame = encoder + .encode_to_message(EncodeInput::Texture((source.0 as _, 0)), 0) + .unwrap(); + check_picture(&mut decoder, frame, width, height, false, 0); + + // Overwrite and release the capture resource before any repeats. + let overwritten = pixels(width, height, 224); + (*context.0).UpdateSubresource( + source.0 as _, + 0, + ptr::null(), + overwritten.as_ptr() as _, + width * 4, + 0, + ); + (*context.0).Flush(); + drop(source); + for count in 1..=100 { + let pts = count * 100; + let frame = + encoder.encode_to_message(EncodeInput::Repeat, pts).unwrap(); + check_picture(&mut decoder, frame, width, height, false, pts); + } + + let source = texture(device.0, width, height, 224); + let frame = encoder + .encode_to_message(EncodeInput::Texture((source.0 as _, 0)), 10100) + .unwrap(); + check_picture(&mut decoder, frame, width, height, true, 10100); + drop(source); + encoder.encoder.set_bitrate(1500).unwrap(); + let frame = encoder + .encode_to_message(EncodeInput::Repeat, 10200) + .unwrap(); + check_picture(&mut decoder, frame, width, height, true, 10200); + eprintln!( + "PASS {:?} {:?} {:?} {width}x{height} adapter={luid:#x}", + vendor, driver, format + ); + tested += 1; + } + } + } + } + assert!(tested > 0, "no hardware encoders were tested"); + eprintln!("Validated {tested} encoder/codec/resolution combinations"); + } +} diff --git a/src/server/video_service.rs b/src/server/video_service.rs index 754c491a7..a13aebb52 100644 --- a/src/server/video_service.rs +++ b/src/server/video_service.rs @@ -64,6 +64,8 @@ use std::{ pub const OPTION_REFRESH: &'static str = "refresh"; +mod static_refresh; + #[cfg(windows)] const DXGI_RECOVERY_LIMIT: usize = 3; #[cfg(windows)] @@ -731,6 +733,16 @@ fn run(vs: VideoService) -> ResultType<()> { let mut encoder_warmed_up = false; let capture_width = c.width; let capture_height = c.height; + let mut static_refresh = static_refresh::StaticRefresh::new( + vs.source, + codec_format, + &encoder_cfg, + &sp, + &recorder, + display_idx, + capture_width, + capture_height, + ); let (mut second_instant, mut send_counter) = (Instant::now(), 0); while sp.ok() { @@ -855,7 +867,8 @@ fn run(vs: VideoService) -> ResultType<()> { } let frame = frame.to(encoder.yuvfmt(), &mut yuv, &mut mid_data)?; - let send_conn_ids = handle_one_frame( + static_refresh.on_frame(&frame); + let result = handle_one_frame( display_idx, &sp, frame, @@ -868,7 +881,8 @@ fn run(vs: VideoService) -> ResultType<()> { capture_width, capture_height, )?; - frame_controller.set_send(now, send_conn_ids); + static_refresh.on_frame_encoded(&result); + frame_controller.set_send(now, result.send_conn_ids); send_counter += 1; } #[cfg(windows)] @@ -932,11 +946,22 @@ fn run(vs: VideoService) -> ResultType<()> { &mut encoder_warmed_up, capture_width, capture_height, - )?; + )? + .send_conn_ids; + static_refresh.on_encoded(!send_conn_ids.is_empty()); frame_controller.set_send(now, send_conn_ids); send_counter += 1; } } + static_refresh.try_encode( + &yuv, + spf, + quality, + now, + ms, + &mut encoder, + &mut frame_controller, + )?; } Err(err) => { #[cfg(windows)] @@ -1261,7 +1286,7 @@ fn handle_one_frame( encoder_warmed_up: &mut bool, width: usize, height: usize, -) -> ResultType> { +) -> ResultType { sp.snapshot(|sps| { // so that new sub and old sub share the same encoder after switch if sps.has_subscribes() { @@ -1272,6 +1297,7 @@ fn handle_one_frame( })?; let mut send_conn_ids: HashSet = Default::default(); + let mut vpx_no_output = false; let first = *first_frame; *first_frame = false; match encoder.encode_to_message(frame, ms) { @@ -1289,6 +1315,7 @@ fn handle_one_frame( send_conn_ids = sp.send_video_frame(msg); } Err(e) => { + vpx_no_output = e.is::(); *encode_fail_counter += 1; // Encoding errors are not frequent except on Android if !cfg!(target_os = "android") { @@ -1319,7 +1346,10 @@ fn handle_one_frame( } } } - Ok(send_conn_ids) + Ok(static_refresh::FrameEncodeResult { + send_conn_ids, + vpx_no_output, + }) } #[inline] diff --git a/src/server/video_service/static_refresh.rs b/src/server/video_service/static_refresh.rs new file mode 100644 index 000000000..f7a7a9295 --- /dev/null +++ b/src/server/video_service/static_refresh.rs @@ -0,0 +1,745 @@ +use super::{GenericService, VideoFrameController, VideoSource}; +use base::message_proto::Message; +use hbb_common::{bail, ResultType}; +use scrap::{ + codec::{Encoder, EncoderCfg, BR_BALANCED, BR_BEST, BR_SPEED}, + record::Recorder, + CodecFormat, EncodeInput, +}; +use std::{ + collections::HashSet, + sync::{Arc, Mutex}, + time::{Duration, Instant}, +}; + +const INITIAL_REPEAT_INTERVAL: Duration = Duration::from_millis(100); +const MAX_REPEAT_FPS: u32 = 30; +const MAX_REPEAT_FAILURES: usize = 3; +const MAX_REPEAT_NO_OUTPUTS: usize = 100; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum HardwareBackend { + Nvenc, + Qsv, + Amf, + Vaapi, + VideoToolbox, + Other, +} + +fn max_repeat_frames(codec: CodecFormat, quality: f32, backend: HardwareBackend) -> usize { + use HardwareBackend::*; + // Measured entries sample quality 0.1, BR_SPEED, BR_BALANCED, and BR_BEST. + // For measured entries, Original values come from AI-run tests. Active values + // were adjusted by comparing 1080p screenshots captured every 10 repeat outputs. + let (low, low_to_balanced, balanced_to_best, best) = match (codec, backend) { + (CodecFormat::VP8, _) => (20, 20, 20, 30), // Original: (20, 20, 20, 30) + (CodecFormat::VP9, _) => (40, 20, 20, 20), // Original: (40, 20, 20, 20) + (CodecFormat::H264, Nvenc) => (80, 50, 60, 30), // Original: (270, 50, 60, 30) + (CodecFormat::H264, Qsv) => (80, 70, 130, 110), // Original: (80, 70, 130, 110) + (CodecFormat::H264, Amf) => (150, 150, 170, 110), // Original: (560, 150, 170, 110) + (CodecFormat::H264, Vaapi) => (150, 100, 90, 60), // Original: (330, 100, 90, 60) + (CodecFormat::H265, Nvenc) => (160, 90, 60, 30), // Original: (160, 90, 60, 30) + (CodecFormat::H265, Qsv) => (150, 30, 30, 30), // Original: (150, 30, 30, 30) + (CodecFormat::H265, Amf) => (200, 200, 200, 120), // Original: (580, 200, 200, 120) + // The capture loop already repeats ten frames to drain VideoToolbox latency. + (CodecFormat::H265, VideoToolbox) => (90, 90, 50, 90), // Original: (230, 90, 50, 90) + (CodecFormat::H264, _) => (90, 60, 60, 30), + (CodecFormat::H265, _) => (90, 60, 60, 30), + (CodecFormat::AV1 | CodecFormat::Unknown, _) => return 0, + }; + if quality >= BR_BEST { + best + } else if quality >= BR_BALANCED { + balanced_to_best + } else if quality >= BR_SPEED { + low_to_balanced + } else { + low + } +} + +#[derive(Debug)] +pub(super) struct FrameEncodeResult { + pub(super) send_conn_ids: HashSet, + pub(super) vpx_no_output: bool, +} + +pub(super) struct StaticRefresh<'a> { + source: VideoSource, + codec_format: CodecFormat, + hardware_backend: HardwareBackend, + sp: &'a GenericService, + recorder: &'a Arc>>, + display_idx: usize, + width: usize, + height: usize, + last_encode: Instant, + #[cfg(test)] + elapsed_since_encode: Duration, + repeat_output_counter: usize, + repeat_no_outputs: usize, + repeat_failures: usize, + source_ready: bool, + #[cfg(all(windows, feature = "vram"))] + has_texture: bool, +} + +impl<'a> StaticRefresh<'a> { + pub(super) fn new( + source: VideoSource, + codec_format: CodecFormat, + encoder_config: &EncoderCfg, + sp: &'a GenericService, + recorder: &'a Arc>>, + display_idx: usize, + width: usize, + height: usize, + ) -> Self { + let hardware_backend = match encoder_config { + #[cfg(feature = "hwcodec")] + EncoderCfg::HWRAM(config) => match config.name.as_str() { + "h264_nvenc" | "hevc_nvenc" => HardwareBackend::Nvenc, + "h264_qsv" | "hevc_qsv" => HardwareBackend::Qsv, + "h264_amf" | "hevc_amf" => HardwareBackend::Amf, + "h264_vaapi" => HardwareBackend::Vaapi, + "hevc_videotoolbox" => HardwareBackend::VideoToolbox, + _ => HardwareBackend::Other, + }, + #[cfg(feature = "vram")] + // The FFMPEG VRAM driver selects the encoder from the adapter vendor. + EncoderCfg::VRAM(config) => match config.feature.vendor { + scrap::vram::Driver::NV => HardwareBackend::Nvenc, + scrap::vram::Driver::MFX => HardwareBackend::Qsv, + scrap::vram::Driver::AMF => HardwareBackend::Amf, + _ => HardwareBackend::Other, + }, + _ => HardwareBackend::Other, + }; + Self { + source, + codec_format, + hardware_backend, + sp, + recorder, + display_idx, + width, + height, + last_encode: Instant::now(), + #[cfg(test)] + elapsed_since_encode: Duration::ZERO, + repeat_output_counter: 0, + repeat_no_outputs: 0, + repeat_failures: 0, + source_ready: false, + #[cfg(all(windows, feature = "vram"))] + has_texture: false, + } + } + + pub(super) fn on_frame(&mut self, _frame: &EncodeInput) { + self.repeat_output_counter = 0; + self.repeat_no_outputs = 0; + self.source_ready = false; + #[cfg(all(windows, feature = "vram"))] + { + self.has_texture = + matches!(_frame, EncodeInput::Texture((texture, 0)) if !texture.is_null()); + } + } + + pub(super) fn on_frame_encoded(&mut self, result: &FrameEncodeResult) { + // VP8/VP9 rate control may drop a frame without an encoding error. + // The latest input is still cached, so allow repeats even without output; + // otherwise the final update may never be sent once capture becomes idle. + self.on_encoded( + !result.send_conn_ids.is_empty() + || (matches!(self.codec_format, CodecFormat::VP8 | CodecFormat::VP9) + && result.vpx_no_output), + ); + } + + pub(super) fn on_encoded(&mut self, success: bool) { + self.last_encode = Instant::now(); + #[cfg(test)] + { + self.elapsed_since_encode = Duration::ZERO; + } + if success { + self.source_ready = true; + self.repeat_failures = 0; + } + } + + pub(super) fn try_encode( + &mut self, + yuv: &[u8], + spf: Duration, + quality: f32, + now: Instant, + ms: i64, + encoder: &mut Encoder, + frame_controller: &mut VideoFrameController, + ) -> ResultType<()> { + let spf = spf.max(Duration::from_secs(1) / MAX_REPEAT_FPS); + if !self.source.is_monitor() + || self.codec_format == CodecFormat::AV1 + || !self.source_ready + || (self.repeat_failures >= MAX_REPEAT_FAILURES + && !matches!( + self.codec_format, + CodecFormat::VP8 | CodecFormat::VP9 | CodecFormat::AV1 + )) + || self.repeat_no_outputs >= MAX_REPEAT_NO_OUTPUTS + || self.repeat_output_counter + >= max_repeat_frames(self.codec_format, quality, self.hardware_backend) + || { + #[cfg(not(test))] + let elapsed = self.last_encode.elapsed(); + #[cfg(test)] + let elapsed = self.elapsed_since_encode; + let interval = if matches!(self.codec_format, CodecFormat::VP8 | CodecFormat::VP9) + || (self.repeat_output_counter == 0 && self.repeat_no_outputs == 0) + { + INITIAL_REPEAT_INTERVAL.max(spf) + } else { + spf + }; + elapsed < interval + } + { + return Ok(()); + } + let frame = if yuv.is_empty() { + None + } else { + Some(EncodeInput::YUV(yuv)) + }; + #[cfg(all(windows, feature = "vram"))] + let frame = frame.or_else(|| self.has_texture.then_some(EncodeInput::Repeat)); + if let Some(frame) = frame { + self.sp.snapshot(|sps| { + if sps.has_subscribes() { + bail!("SWITCH"); + } + Ok(()) + })?; + + let result = encoder.encode_to_message(frame, ms); + self.last_encode = if matches!(self.codec_format, CodecFormat::VP8 | CodecFormat::VP9) { + Instant::now() + } else { + // Align with the capture loop so encoding time does not skip the next tick. + now + }; + #[cfg(test)] + { + self.elapsed_since_encode = Duration::ZERO; + } + let mut vf = match result { + Ok(vf) if vf.union.is_none() => { + self.repeat_no_outputs += 1; + return Ok(()); + } + Ok(vf) => vf, + Err(error) => { + self.repeat_no_outputs += 1; + self.repeat_failures += 1; + if error.is::() { + hbb_common::throttled_log!( + Duration::from_secs(60), + trace, + "static refresh produced no output: {error:?}" + ); + } else { + hbb_common::throttled_log!( + Duration::from_secs(60), + debug, + "static refresh failed ({}): {error:?}", + self.repeat_failures + ); + } + return Ok(()); + } + }; + self.repeat_output_counter += 1; + self.repeat_failures = 0; + vf.display = self.display_idx as _; + let mut msg = Message::new(); + msg.set_video_frame(vf); + self.recorder + .lock() + .unwrap() + .as_mut() + .map(|r| r.write_message(&msg, self.width, self.height)); + let send_conn_ids = self.sp.send_video_frame(msg); + frame_controller.set_send(now, send_conn_ids); + // Static refinement must not count as motion for adaptive bitrate. + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::server::{connection::ConnInner, service::Service}; + use base::message_proto::VideoFrame; + use hbb_common::tokio::sync::mpsc; + use scrap::{ + codec::{EncoderApi, EncoderCfg, BR_BALANCED}, + vpxcodec::{VpxEncoderConfig, VpxVideoCodecId}, + EncodeYuvFormat, + }; + use std::{cell::Cell, rc::Rc}; + + pub(super) fn new_refresh<'a>( + source: VideoSource, + codec_format: CodecFormat, + sp: &'a GenericService, + recorder: &'a Arc>>, + display_idx: usize, + width: usize, + height: usize, + ) -> StaticRefresh<'a> { + // The mock encoders simulate each format without a hardware backend. + let config = EncoderCfg::VPX(VpxEncoderConfig { + width: width as _, + height: height as _, + quality: BR_BALANCED, + codec: VpxVideoCodecId::VP9, + keyframe_interval: None, + }); + StaticRefresh::new( + source, + codec_format, + &config, + sp, + recorder, + display_idx, + width, + height, + ) + } + + struct CountingEncoder(Rc>); + + impl EncoderApi for CountingEncoder { + fn new(_: EncoderCfg, _: bool) -> ResultType { + unreachable!() + } + + fn encode_to_message(&mut self, frame: EncodeInput, _: i64) -> ResultType { + match frame { + EncodeInput::YUV(yuv) => assert_eq!(yuv, &[1]), + #[cfg(all(windows, feature = "vram"))] + EncodeInput::Repeat => {} + _ => panic!("unexpected encoder input"), + } + self.0.set(self.0.get() + 1); + let mut vf = VideoFrame::new(); + vf.set_h264s(Default::default()); + Ok(vf) + } + + fn yuvfmt(&self) -> EncodeYuvFormat { + unreachable!() + } + + #[cfg(feature = "vram")] + fn input_texture(&self) -> bool { + false + } + + fn set_quality(&mut self, _: f32) -> ResultType<()> { + unreachable!() + } + + fn bitrate(&self) -> u32 { + 0 + } + + fn support_changing_quality(&self) -> bool { + false + } + + fn latency_free(&self) -> bool { + true + } + + fn is_hardware(&self) -> bool { + false + } + + fn disable(&self) { + unreachable!() + } + } + + fn attempt( + refresh: &mut StaticRefresh<'_>, + calls: &Rc>, + yuv: &[u8], + spf: Duration, + ) { + let mut encoder = Encoder { + codec: Box::new(CountingEncoder(Rc::clone(calls))), + }; + refresh + .try_encode( + yuv, + spf, + BR_BALANCED, + Instant::now(), + 0, + &mut encoder, + &mut VideoFrameController::new(0), + ) + .unwrap(); + } + + #[test] + fn long_network_waits_preserve_budget_and_new_frames_restart_it() { + let sp = GenericService::new("static-refresh-test".to_owned(), false); + let recorder = Arc::new(Mutex::new(None)); + let mut refresh = new_refresh( + VideoSource::Monitor, + CodecFormat::VP9, + &sp, + &recorder, + 0, + 1, + 1, + ); + let calls = Rc::new(Cell::new(0)); + let spf = Duration::from_millis(100); + refresh.on_frame(&EncodeInput::YUV(&[1])); + refresh.on_encoded(true); + let limit = max_repeat_frames(CodecFormat::VP9, BR_BALANCED, HardwareBackend::Other); + for _ in 0..limit { + refresh.elapsed_since_encode = Duration::from_secs(60); + attempt(&mut refresh, &calls, &[1], spf); + } + assert_eq!(calls.get(), limit); + + refresh.elapsed_since_encode = Duration::from_secs(600); + attempt(&mut refresh, &calls, &[1], spf); + assert_eq!(calls.get(), limit); + + refresh.on_frame(&EncodeInput::YUV(&[1])); + refresh.on_encoded(true); + refresh.elapsed_since_encode = Duration::from_secs(1); + attempt(&mut refresh, &calls, &[1], spf); + assert_eq!(calls.get(), limit + 1); + } + + #[test] + fn initial_delay_repeat_interval_and_qos_limit_refresh_rate() { + let sp = GenericService::new("static-refresh-test".to_owned(), false); + let recorder = Arc::new(Mutex::new(None)); + let mut refresh = new_refresh( + VideoSource::Monitor, + CodecFormat::H264, + &sp, + &recorder, + 0, + 1, + 1, + ); + let calls = Rc::new(Cell::new(0)); + refresh.on_frame(&EncodeInput::YUV(&[1])); + refresh.on_encoded(true); + attempt(&mut refresh, &calls, &[1], Duration::from_millis(10)); + assert_eq!(calls.get(), 0); + + refresh.elapsed_since_encode = Duration::from_millis(50); + attempt(&mut refresh, &calls, &[1], Duration::from_millis(10)); + assert_eq!(calls.get(), 0); + + refresh.elapsed_since_encode = Duration::from_millis(100); + attempt(&mut refresh, &calls, &[1], Duration::from_millis(10)); + assert_eq!(calls.get(), 1); + + refresh.elapsed_since_encode = Duration::from_millis(33); + attempt(&mut refresh, &calls, &[1], Duration::from_millis(10)); + assert_eq!(calls.get(), 1); + + refresh.elapsed_since_encode = Duration::from_secs(1) / 30; + attempt(&mut refresh, &calls, &[1], Duration::from_millis(10)); + assert_eq!(calls.get(), 2); + + refresh.elapsed_since_encode = Duration::from_millis(49); + attempt(&mut refresh, &calls, &[1], Duration::from_millis(50)); + assert_eq!(calls.get(), 2); + refresh.elapsed_since_encode = Duration::from_millis(50); + attempt(&mut refresh, &calls, &[1], Duration::from_millis(50)); + assert_eq!(calls.get(), 3); + + refresh.on_frame(&EncodeInput::YUV(&[1])); + refresh.on_encoded(true); + refresh.elapsed_since_encode = Duration::from_millis(50); + attempt(&mut refresh, &calls, &[1], Duration::from_millis(10)); + assert_eq!(calls.get(), 3); + + refresh.elapsed_since_encode = Duration::from_secs(1); + attempt(&mut refresh, &calls, &[1], Duration::from_secs(2)); + assert_eq!(calls.get(), 3); + + refresh.elapsed_since_encode = Duration::from_secs(3); + attempt(&mut refresh, &calls, &[1], Duration::from_secs(2)); + assert_eq!(calls.get(), 4); + attempt(&mut refresh, &calls, &[1], Duration::from_secs(2)); + assert_eq!(calls.get(), 4); + refresh.elapsed_since_encode = Duration::from_millis(1999); + attempt(&mut refresh, &calls, &[1], Duration::from_secs(2)); + assert_eq!(calls.get(), 4); + refresh.elapsed_since_encode = Duration::from_millis(2000); + attempt(&mut refresh, &calls, &[1], Duration::from_secs(2)); + assert_eq!(calls.get(), 5); + } + + #[test] + #[cfg(all(windows, feature = "vram"))] + fn texture_frames_request_repeats_without_accessing_capture_resources() { + let sp = GenericService::new("static-refresh-test".to_owned(), false); + let recorder = Arc::new(Mutex::new(None)); + let mut refresh = new_refresh( + VideoSource::Monitor, + CodecFormat::H264, + &sp, + &recorder, + 0, + 1, + 1, + ); + let calls = Rc::new(Cell::new(0)); + // Deliberately not a D3D resource: neither hook may dereference it. + let texture = std::ptr::NonNull::::dangling().as_ptr() as _; + for _ in 0..60 { + refresh.on_frame(&EncodeInput::Texture((texture, 0))); + } + refresh.on_encoded(true); + refresh.elapsed_since_encode = Duration::from_secs(1); + attempt(&mut refresh, &calls, &[], Duration::from_millis(16)); + assert_eq!(calls.get(), 1); + + for frame in [ + EncodeInput::YUV(&[1]), + EncodeInput::Texture((texture, 90)), + EncodeInput::Texture((std::ptr::null_mut(), 0)), + ] { + refresh.on_frame(&frame); + refresh.on_encoded(true); + refresh.elapsed_since_encode = Duration::from_secs(1); + attempt(&mut refresh, &calls, &[], Duration::from_millis(16)); + assert_eq!(calls.get(), 1); + } + } + + #[test] + fn cameras_av1_and_missing_frames_are_not_refreshed() { + let sp = GenericService::new("static-refresh-test".to_owned(), false); + let recorder = Arc::new(Mutex::new(None)); + let calls = Rc::new(Cell::new(0)); + for (source, codec_format, yuv) in [ + (VideoSource::Camera, CodecFormat::VP9, &[1][..]), + (VideoSource::Monitor, CodecFormat::AV1, &[1][..]), + (VideoSource::Monitor, CodecFormat::VP9, &[][..]), + ] { + let mut refresh = new_refresh(source, codec_format, &sp, &recorder, 0, 1, 1); + refresh.on_frame(&EncodeInput::YUV(yuv)); + refresh.on_encoded(true); + refresh.elapsed_since_encode = Duration::from_secs(60); + attempt(&mut refresh, &calls, yuv, Duration::from_millis(100)); + } + assert_eq!(calls.get(), 0); + } + + fn assert_output_budget( + refresh: &mut StaticRefresh<'_>, + sp: &GenericService, + quality: f32, + expected: usize, + ) { + let (tx, rx) = mpsc::unbounded_channel(); + sp.on_subscribe(ConnInner::new(1, None, Some(tx))); + let mut encoder = Encoder { + codec: Box::new(CountingEncoder(Rc::new(Cell::new(0)))), + }; + refresh.on_frame(&EncodeInput::YUV(&[1])); + refresh.on_encoded(true); + for _ in 0..=expected { + refresh.elapsed_since_encode = Duration::from_secs(1); + refresh + .try_encode( + &[1], + Duration::from_millis(33), + quality, + Instant::now(), + 0, + &mut encoder, + &mut VideoFrameController::new(0), + ) + .unwrap(); + } + assert_eq!(rx.len(), expected); + sp.on_unsubscribe(1); + } + + #[test] + fn quality_bands_stop_each_backend_at_its_output_budget() { + let sp = GenericService::new("repeat-quality-test".to_owned(), false); + let recorder = Arc::new(Mutex::new(None)); + assert_eq!( + [0.1, BR_SPEED, BR_BALANCED, BR_BEST] + .map(|q| { max_repeat_frames(CodecFormat::H264, q, HardwareBackend::Vaapi) }), + [150, 100, 90, 60] + ); + for codec in [ + CodecFormat::VP8, + CodecFormat::VP9, + CodecFormat::H264, + CodecFormat::H265, + ] { + for backend in [ + HardwareBackend::Nvenc, + HardwareBackend::Qsv, + HardwareBackend::Amf, + HardwareBackend::Vaapi, + HardwareBackend::VideoToolbox, + HardwareBackend::Other, + ] { + let mut refresh = new_refresh(VideoSource::Monitor, codec, &sp, &recorder, 0, 1, 1); + refresh.hardware_backend = backend; + for quality in [ + 0.1, + BR_BEST, + BR_SPEED, + BR_BALANCED, + BR_SPEED + 0.01, + BR_BEST + 0.1, + ] { + let limit = max_repeat_frames(codec, quality, backend); + assert_output_budget(&mut refresh, &sp, quality, limit); + } + } + } + } + + #[test] + fn speed_boundary_uses_the_speed_preset_for_all_backends() { + for codec in [ + CodecFormat::VP8, + CodecFormat::VP9, + CodecFormat::H264, + CodecFormat::H265, + ] { + for backend in [ + HardwareBackend::Nvenc, + HardwareBackend::Qsv, + HardwareBackend::Amf, + HardwareBackend::Vaapi, + HardwareBackend::VideoToolbox, + HardwareBackend::Other, + ] { + assert_eq!( + max_repeat_frames(codec, BR_SPEED, backend), + max_repeat_frames(codec, BR_SPEED + 0.01, backend), + "{codec:?} {backend:?}" + ); + } + } + } + + #[test] + #[cfg(feature = "hwcodec")] + fn ram_config_selects_the_actual_backend_and_preserves_other_encoders() { + let sp = GenericService::new("repeat-backend-test".to_owned(), false); + let recorder = Arc::new(Mutex::new(None)); + for (name, expected) in [ + ("h264_nvenc", HardwareBackend::Nvenc), + ("hevc_nvenc", HardwareBackend::Nvenc), + ("h264_qsv", HardwareBackend::Qsv), + ("hevc_qsv", HardwareBackend::Qsv), + ("h264_amf", HardwareBackend::Amf), + ("hevc_amf", HardwareBackend::Amf), + ("h264_vaapi", HardwareBackend::Vaapi), + ("hevc_vaapi", HardwareBackend::Other), + ("hevc_videotoolbox", HardwareBackend::VideoToolbox), + ("h264_videotoolbox", HardwareBackend::Other), + ] { + let config = EncoderCfg::HWRAM(scrap::hwcodec::HwRamEncoderConfig { + name: name.to_owned(), + mc_name: None, + width: 1920, + height: 1080, + quality: BR_BALANCED, + keyframe_interval: None, + }); + let codec_format = if name.starts_with("hevc_") { + CodecFormat::H265 + } else { + CodecFormat::H264 + }; + let mut refresh = StaticRefresh::new( + VideoSource::Monitor, + codec_format, + &config, + &sp, + &recorder, + 0, + 1920, + 1080, + ); + for quality in [0.1, BR_SPEED, BR_BALANCED, BR_BEST] { + let limit = max_repeat_frames(codec_format, quality, expected); + assert_output_budget(&mut refresh, &sp, quality, limit); + } + } + } + + #[test] + #[cfg(feature = "vram")] + fn vram_config_resolves_ffmpeg_to_the_actual_hardware_backend() { + let sp = GenericService::new("repeat-vram-backend-test".to_owned(), false); + let recorder = Arc::new(Mutex::new(None)); + for (vendor, expected) in [ + ("NV", HardwareBackend::Nvenc), + ("MFX", HardwareBackend::Qsv), + ("AMF", HardwareBackend::Amf), + ("FFMPEG", HardwareBackend::Other), + ] { + let config = EncoderCfg::VRAM(scrap::vram::VRamEncoderConfig { + device: Default::default(), + width: 1920, + height: 1080, + quality: BR_BALANCED, + keyframe_interval: None, + feature: serde_json::from_value(serde_json::json!({ + "driver": "FFMPEG", + "vendor": vendor, + "luid": 0, + "data_format": "H264", + })) + .unwrap(), + }); + let mut refresh = StaticRefresh::new( + VideoSource::Monitor, + CodecFormat::H264, + &config, + &sp, + &recorder, + 0, + 1920, + 1080, + ); + for quality in [0.1, BR_SPEED, BR_BALANCED, BR_BEST] { + let limit = max_repeat_frames(CodecFormat::H264, quality, expected); + assert_output_budget(&mut refresh, &sp, quality, limit); + } + } + } +} + +#[cfg(test)] +mod error_tests; diff --git a/src/server/video_service/static_refresh/error_tests.rs b/src/server/video_service/static_refresh/error_tests.rs new file mode 100644 index 000000000..50eb3e25f --- /dev/null +++ b/src/server/video_service/static_refresh/error_tests.rs @@ -0,0 +1,579 @@ +use super::{tests::new_refresh, *}; +use crate::server::{connection::ConnInner, service::Service}; +use base::message_proto::VideoFrame; +use hbb_common::tokio::sync::mpsc; +use scrap::{ + codec::{EncoderApi, EncoderCfg, BR_BALANCED}, + EncodeYuvFormat, +}; +use std::{cell::RefCell, collections::VecDeque, rc::Rc}; + +#[derive(Clone, Copy)] +enum Outcome { + Frame, + Empty, + Dropped, + Error, +} + +struct State { + outcomes: VecDeque, + calls: usize, + disabled: usize, + inputs: Vec>, +} + +struct ScriptedEncoder(Rc>); + +impl EncoderApi for ScriptedEncoder { + fn new(_: EncoderCfg, _: bool) -> ResultType { + unreachable!() + } + fn encode_to_message(&mut self, input: EncodeInput, _: i64) -> ResultType { + let mut state = self.0.borrow_mut(); + state.calls += 1; + if let EncodeInput::YUV(yuv) = input { + state.inputs.push(yuv.to_vec()); + } + match state.outcomes.pop_front().expect("unexpected encode") { + Outcome::Frame => { + let mut vf = VideoFrame::new(); + vf.set_h264s(Default::default()); + Ok(vf) + } + Outcome::Empty => Ok(VideoFrame::new()), + Outcome::Dropped => Err(scrap::VpxNoOutput.into()), + Outcome::Error => bail!("injected encoding error"), + } + } + fn yuvfmt(&self) -> EncodeYuvFormat { + unreachable!() + } + #[cfg(feature = "vram")] + fn input_texture(&self) -> bool { + false + } + fn set_quality(&mut self, _: f32) -> ResultType<()> { + unreachable!() + } + fn bitrate(&self) -> u32 { + 0 + } + fn support_changing_quality(&self) -> bool { + false + } + fn latency_free(&self) -> bool { + true + } + fn is_hardware(&self) -> bool { + true + } + fn disable(&self) { + self.0.borrow_mut().disabled += 1; + } +} + +fn encoder(outcomes: &[Outcome]) -> (Encoder, Rc>) { + let state = Rc::new(RefCell::new(State { + outcomes: outcomes.iter().copied().collect(), + calls: 0, + disabled: 0, + inputs: Vec::new(), + })); + ( + Encoder { + codec: Box::new(ScriptedEncoder(Rc::clone(&state))), + }, + state, + ) +} + +fn attempt(refresh: &mut StaticRefresh<'_>, encoder: &mut Encoder) { + refresh.elapsed_since_encode = Duration::from_secs(1); + refresh + .try_encode( + &[1], + Duration::from_millis(100), + BR_BALANCED, + Instant::now(), + 0, + encoder, + &mut VideoFrameController::new(0), + ) + .unwrap(); +} + +#[test] +fn failures_pause_refresh_until_a_real_frame_succeeds() { + use Outcome::*; + let sp = GenericService::new("repeat-errors-test".to_owned(), false); + let (tx, rx) = mpsc::unbounded_channel(); + sp.on_subscribe(ConnInner::new(1, None, Some(tx))); + let recorder = Arc::new(Mutex::new(None)); + let mut refresh = new_refresh( + VideoSource::Monitor, + CodecFormat::H264, + &sp, + &recorder, + 0, + 1, + 1, + ); + let (mut encoder, state) = encoder(&[Error, Error, Empty, Frame, Error, Error, Error, Frame]); + refresh.on_frame(&EncodeInput::YUV(&[1])); + refresh.on_encoded(true); + attempt(&mut refresh, &mut encoder); + refresh + .try_encode( + &[1], + Duration::from_millis(100), + BR_BALANCED, + Instant::now(), + 0, + &mut encoder, + &mut VideoFrameController::new(0), + ) + .unwrap(); + assert_eq!( + state.borrow().calls, + 1, + "failure must retain the rate limit" + ); + attempt(&mut refresh, &mut encoder); + attempt(&mut refresh, &mut encoder); + assert_eq!(rx.len(), 0); + attempt(&mut refresh, &mut encoder); + assert_eq!(rx.len(), 1, "empty output must not stop further refreshes"); + for _ in 0..3 { + attempt(&mut refresh, &mut encoder); + } + attempt(&mut refresh, &mut encoder); + assert_eq!(state.borrow().calls, 7); + assert_eq!(rx.len(), 1); + assert_eq!(state.borrow().disabled, 0); + + refresh.on_frame(&EncodeInput::YUV(&[1])); + refresh.on_encoded(false); + attempt(&mut refresh, &mut encoder); + assert_eq!( + state.borrow().calls, + 7, + "a failed real frame must not restart refresh" + ); + refresh.on_frame(&EncodeInput::YUV(&[1])); + refresh.on_encoded(true); + attempt(&mut refresh, &mut encoder); + assert_eq!(state.borrow().calls, 8); + assert_eq!(rx.len(), 2); + assert_eq!(state.borrow().disabled, 0); +} + +#[test] +fn empty_outputs_exhaust_the_no_output_budget_without_disabling_the_encoder() { + let sp = GenericService::new("repeat-empty-test".to_owned(), false); + let recorder = Arc::new(Mutex::new(None)); + let mut refresh = new_refresh( + VideoSource::Monitor, + CodecFormat::H264, + &sp, + &recorder, + 0, + 1, + 1, + ); + let (mut encoder, state) = encoder(&[Outcome::Empty; 100]); + refresh.on_frame(&EncodeInput::YUV(&[1])); + refresh.on_encoded(true); + for _ in 0..101 { + attempt(&mut refresh, &mut encoder); + } + assert_eq!(state.borrow().calls, 100); + assert_eq!(state.borrow().disabled, 0); +} + +#[test] +fn vpx_drops_exhaust_the_no_output_budget_without_disabling_the_encoder() { + let sp = GenericService::new("repeat-vpx-drops-test".to_owned(), false); + let recorder = Arc::new(Mutex::new(None)); + for codec in [CodecFormat::VP8, CodecFormat::VP9] { + let mut refresh = new_refresh(VideoSource::Monitor, codec, &sp, &recorder, 0, 1, 1); + let limit = MAX_REPEAT_NO_OUTPUTS; + let (mut encoder, state) = encoder(&vec![Outcome::Dropped; limit]); + refresh.on_frame(&EncodeInput::YUV(&[1])); + refresh.on_encoded(true); + for _ in 0..=limit { + attempt(&mut refresh, &mut encoder); + } + assert_eq!(state.borrow().calls, limit, "{codec:?}"); + assert_eq!(state.borrow().disabled, 0); + } +} + +#[test] +fn vpx_dropped_capture_repeats_the_latest_frame() { + use super::super::handle_one_frame; + use Outcome::*; + let sp = GenericService::new("repeat-dropped-capture-test".to_owned(), false); + let recorder = Arc::new(Mutex::new(None)); + for codec in [CodecFormat::VP8, CodecFormat::VP9] { + let (tx, mut rx) = mpsc::unbounded_channel(); + sp.on_subscribe(ConnInner::new(1, None, Some(tx))); + let mut refresh = new_refresh(VideoSource::Monitor, codec, &sp, &recorder, 0, 1, 1); + let (mut encoder, state) = encoder(&[Frame, Dropped, Dropped, Frame]); + refresh.on_frame(&EncodeInput::YUV(&[1])); + refresh.on_encoded(true); + attempt(&mut refresh, &mut encoder); + assert!(rx.try_recv().unwrap().1.has_video_frame()); + + let latest_yuv = [2]; + refresh.on_frame(&EncodeInput::YUV(&latest_yuv)); + let mut normal_failures = 0; + let mut first_frame = false; + let mut encoder_warmed_up = false; + let result = handle_one_frame( + 0, + &sp, + EncodeInput::YUV(&latest_yuv), + 100, + &mut encoder, + recorder.clone(), + &mut normal_failures, + &mut first_frame, + &mut encoder_warmed_up, + 1, + 1, + ) + .unwrap(); + assert!(result.send_conn_ids.is_empty()); + assert!(result.vpx_no_output); + refresh.on_frame_encoded(&result); + assert_eq!(rx.len(), 0); + for (elapsed_ms, expected_calls, expected_outputs) in + [(99, 2, 0), (100, 3, 0), (100, 4, 1)] + { + refresh.elapsed_since_encode = Duration::from_millis(elapsed_ms); + refresh + .try_encode( + &latest_yuv, + Duration::from_nanos(33_333_333), + BR_BALANCED, + Instant::now(), + expected_calls as i64 * 100, + &mut encoder, + &mut VideoFrameController::new(0), + ) + .unwrap(); + assert_eq!(state.borrow().calls, expected_calls, "{codec:?}"); + assert_eq!(rx.len(), expected_outputs); + } + assert_eq!(state.borrow().inputs, vec![vec![1], vec![2], vec![2], vec![2]]); + assert_eq!(state.borrow().disabled, 0); + sp.on_unsubscribe(1); + } +} + +#[test] +fn captured_frame_errors_do_not_enable_repeat() { + use super::super::handle_one_frame; + let sp = GenericService::new("repeat-capture-error-test".to_owned(), false); + let recorder = Arc::new(Mutex::new(None)); + for (codec, outcome, vpx_no_output) in [ + (CodecFormat::VP8, Outcome::Error, false), + (CodecFormat::VP9, Outcome::Error, false), + (CodecFormat::H264, Outcome::Dropped, true), + (CodecFormat::H265, Outcome::Dropped, true), + ] { + let mut refresh = new_refresh(VideoSource::Monitor, codec, &sp, &recorder, 0, 1, 1); + let (mut encoder, state) = encoder(&[outcome, Outcome::Frame]); + refresh.on_frame(&EncodeInput::YUV(&[1])); + let mut normal_failures = 0; + let mut first_frame = false; + let mut encoder_warmed_up = false; + let result = handle_one_frame( + 0, + &sp, + EncodeInput::YUV(&[1]), + 0, + &mut encoder, + recorder.clone(), + &mut normal_failures, + &mut first_frame, + &mut encoder_warmed_up, + 1, + 1, + ) + .unwrap(); + assert!(result.send_conn_ids.is_empty()); + assert_eq!(result.vpx_no_output, vpx_no_output); + refresh.on_frame_encoded(&result); + attempt(&mut refresh, &mut encoder); + assert_eq!(state.borrow().calls, 1, "{codec:?}"); + assert_eq!(state.borrow().disabled, 0); + } +} + +#[test] +fn unsuccessful_repeats_preserve_the_frame_budget() { + let sp = GenericService::new("repeat-frame-budget-test".to_owned(), false); + let recorder = Arc::new(Mutex::new(None)); + for codec in [CodecFormat::VP8, CodecFormat::VP9] { + let (tx, rx) = mpsc::unbounded_channel(); + sp.on_subscribe(ConnInner::new(1, None, Some(tx))); + let mut refresh = new_refresh(VideoSource::Monitor, codec, &sp, &recorder, 0, 1, 1); + let limit = max_repeat_frames(codec, BR_BALANCED, HardwareBackend::Other); + let mut outcomes = vec![Outcome::Dropped; 40]; + outcomes.extend(vec![Outcome::Empty; 40]); + outcomes.extend(vec![Outcome::Frame; limit + 1]); + let (mut encoder, state) = encoder(&outcomes); + refresh.on_frame(&EncodeInput::YUV(&[1])); + refresh.on_encoded(true); + for _ in 0..80 { + attempt(&mut refresh, &mut encoder); + } + assert_eq!(rx.len(), 0, "{codec:?}"); + for _ in 0..=limit { + attempt(&mut refresh, &mut encoder); + } + assert_eq!(state.borrow().calls, 80 + limit, "{codec:?}"); + assert_eq!(rx.len(), limit); + assert_eq!(state.borrow().disabled, 0); + sp.on_unsubscribe(1); + } +} + +#[test] +fn repeat_intervals_are_codec_specific_after_the_first_attempt() { + let sp = GenericService::new("repeat-interval-test".to_owned(), false); + let recorder = Arc::new(Mutex::new(None)); + for (codec, spf_ms, first_interval_ms, repeat_interval_ms) in [ + (CodecFormat::VP8, 10, 100, 100), + (CodecFormat::VP9, 10, 100, 100), + (CodecFormat::VP8, 200, 200, 200), + (CodecFormat::VP9, 200, 200, 200), + (CodecFormat::H264, 10, 100, 34), + (CodecFormat::H265, 10, 100, 34), + (CodecFormat::H264, 16, 100, 34), + (CodecFormat::H265, 16, 100, 34), + (CodecFormat::H264, 50, 100, 50), + (CodecFormat::H265, 50, 100, 50), + (CodecFormat::H264, 200, 200, 200), + (CodecFormat::H265, 200, 200, 200), + ] { + for first in [ + Outcome::Frame, + Outcome::Empty, + Outcome::Dropped, + Outcome::Error, + ] { + let mut refresh = new_refresh(VideoSource::Monitor, codec, &sp, &recorder, 0, 1, 1); + let (mut encoder, state) = encoder(&[first, Outcome::Frame]); + refresh.on_frame(&EncodeInput::YUV(&[1])); + refresh.on_encoded(true); + for (elapsed_ms, expected_calls) in [ + (first_interval_ms - 1, 0), + (first_interval_ms, 1), + (repeat_interval_ms - 1, 1), + (repeat_interval_ms, 2), + ] { + refresh.elapsed_since_encode = Duration::from_millis(elapsed_ms); + refresh + .try_encode( + &[1], + Duration::from_millis(spf_ms), + BR_BALANCED, + Instant::now(), + 0, + &mut encoder, + &mut VideoFrameController::new(0), + ) + .unwrap(); + assert_eq!(state.borrow().calls, expected_calls, "{codec:?}"); + } + } + } +} + +#[test] +fn repeat_timing_uses_the_capture_loop_start() { + let sp = GenericService::new("repeat-loop-clock-test".to_owned(), false); + let recorder = Arc::new(Mutex::new(None)); + let spf = Duration::from_nanos(33_333_333); + let clock_start = Instant::now() + Duration::from_secs(1); + for first in [ + Outcome::Frame, + Outcome::Empty, + Outcome::Dropped, + Outcome::Error, + ] { + let mut refresh = new_refresh( + VideoSource::Monitor, + CodecFormat::H264, + &sp, + &recorder, + 0, + 1, + 1, + ); + let (mut encoder, state) = encoder(&[first, Outcome::Frame]); + refresh.on_frame(&EncodeInput::YUV(&[1])); + refresh.on_encoded(true); + refresh.last_encode = clock_start; + for (index, loop_start) in [ + clock_start + INITIAL_REPEAT_INTERVAL, + clock_start + INITIAL_REPEAT_INTERVAL + spf, + ] + .into_iter() + .enumerate() + { + // Capture processing finishes partway through the current loop. + let capture_end = loop_start + Duration::from_millis(5); + refresh.elapsed_since_encode = capture_end.duration_since(refresh.last_encode); + refresh + .try_encode( + &[1], + spf, + BR_BALANCED, + loop_start, + 0, + &mut encoder, + &mut VideoFrameController::new(0), + ) + .unwrap(); + assert_eq!(state.borrow().calls, index + 1); + } + } +} + +#[test] +fn no_output_budget_is_cumulative_and_restarts_with_a_new_frame() { + let sp = GenericService::new("repeat-no-output-budget-test".to_owned(), false); + let recorder = Arc::new(Mutex::new(None)); + for codec in [CodecFormat::VP8, CodecFormat::VP9] { + let mut refresh = new_refresh(VideoSource::Monitor, codec, &sp, &recorder, 0, 1, 1); + let mut outcomes = Vec::new(); + for _ in 0..10 { + outcomes.push(Outcome::Frame); + for _ in 0..5 { + outcomes.extend([Outcome::Empty, Outcome::Dropped]); + } + } + outcomes.push(Outcome::Empty); + outcomes.push(Outcome::Frame); + let (mut encoder, state) = encoder(&outcomes); + refresh.on_frame(&EncodeInput::YUV(&[1])); + refresh.on_encoded(true); + for _ in 0..111 { + attempt(&mut refresh, &mut encoder); + } + assert_eq!(state.borrow().calls, 110, "{codec:?}"); + assert_eq!(state.borrow().disabled, 0); + + refresh.on_frame(&EncodeInput::YUV(&[1])); + refresh.on_encoded(false); + attempt(&mut refresh, &mut encoder); + assert_eq!(state.borrow().calls, 110); + refresh.on_encoded(true); + attempt(&mut refresh, &mut encoder); + assert_eq!(state.borrow().calls, 111); + attempt(&mut refresh, &mut encoder); + assert_eq!(state.borrow().calls, 112); + } +} + +#[test] +fn repeat_error_limits_are_codec_specific() { + let sp = GenericService::new("repeat-codec-errors-test".to_owned(), false); + let recorder = Arc::new(Mutex::new(None)); + for (codec, expected_calls) in [ + (CodecFormat::VP8, 100), + (CodecFormat::VP9, 100), + (CodecFormat::AV1, 0), + (CodecFormat::H264, 3), + (CodecFormat::H265, 3), + ] { + let mut refresh = new_refresh(VideoSource::Monitor, codec, &sp, &recorder, 0, 1, 1); + let limit = MAX_REPEAT_NO_OUTPUTS; + let (mut encoder, state) = encoder(&vec![Outcome::Error; limit]); + refresh.on_frame(&EncodeInput::YUV(&[1])); + refresh.on_encoded(true); + for _ in 0..=limit { + attempt(&mut refresh, &mut encoder); + } + assert_eq!(state.borrow().calls, expected_calls, "{codec:?}"); + assert_eq!(state.borrow().disabled, 0); + } +} + +#[test] +fn repeats_leave_normal_failure_tracking_and_switching_intact() { + use super::super::handle_one_frame; + use Outcome::*; + let sp = GenericService::new("repeat-normal-errors-test".to_owned(), false); + let recorder = Arc::new(Mutex::new(None)); + let mut refresh = new_refresh( + VideoSource::Monitor, + CodecFormat::H264, + &sp, + &recorder, + 0, + 1, + 1, + ); + let normal_limit = if cfg!(target_os = "android") { 9 } else { 3 }; + let mut outcomes = vec![Error, Error, Frame]; + outcomes.extend(vec![Error; normal_limit - 1]); + let (mut encoder, state) = encoder(&outcomes); + refresh.on_frame(&EncodeInput::YUV(&[1])); + refresh.on_encoded(true); + let mut normal_failures = 0; + let mut first_frame = false; + let mut encoder_warmed_up = false; + handle_one_frame( + 0, + &sp, + EncodeInput::YUV(&[1]), + 0, + &mut encoder, + recorder.clone(), + &mut normal_failures, + &mut first_frame, + &mut encoder_warmed_up, + 1, + 1, + ) + .unwrap(); + attempt(&mut refresh, &mut encoder); + attempt(&mut refresh, &mut encoder); + assert_eq!(state.borrow().disabled, 0); + for _ in 2..normal_limit { + handle_one_frame( + 0, + &sp, + EncodeInput::YUV(&[1]), + 0, + &mut encoder, + recorder.clone(), + &mut normal_failures, + &mut first_frame, + &mut encoder_warmed_up, + 1, + 1, + ) + .unwrap(); + } + let result = handle_one_frame( + 0, + &sp, + EncodeInput::YUV(&[1]), + 0, + &mut encoder, + recorder.clone(), + &mut normal_failures, + &mut first_frame, + &mut encoder_warmed_up, + 1, + 1, + ); + assert_eq!(result.unwrap_err().to_string(), "SWITCH"); + assert_eq!(state.borrow().disabled, 1); +}