mirror of
https://github.com/rustdesk/rustdesk.git
synced 2026-10-19 08:15:03 +00:00
fix(video): improve clarity after scrolling by re-encoding static frames (#16128)
* fix(video): re-encode static frames for up to 10 seconds to improve clarity Signed-off-by: 21pages <sunboeasy@gmail.com> fix(video): limit static re-encoding to 300 frames instead of 10 seconds Signed-off-by: 21pages <sunboeasy@gmail.com> refactor(video): move static frame refresh into a dedicated module Signed-off-by: 21pages <sunboeasy@gmail.com> fix(video): preserve stuck-encoder detection state across static repeats test(video): cover cached texture lifetime across capture restarts fix(video): reduce static re-encoding limit to 100 frames fix(qos): lower qos_trace logging to trace level Signed-off-by: 21pages <sunboeasy@gmail.com> fix(video): skip static frame re-encoding for AV1 perf(video): reuse VRAM encoder input for static refresh Remove the DXGI repeat texture cache and use hwcodec's repeat entry point to avoid copying each captured frame. Isolate static refresh failures, pause after three errors, and resume after a real frame is sent successfully. Cover cached input ownership, repeat failure recovery, refresh limits, and normal encoder failover with regression tests. fix(vram): ignore cached SDK codec backends chore(deps): temporarily use hwcodec re-encode branch perf(video): wait 200ms before starting static refresh chore(deps): update hwcodec for FFI exception handling chore(deps): update hwcodec to 0.8.0 perf(video): start static refresh after 100ms chore(deps): update hwcodec for exception-safe cleanup chore(deps): update hwcodec for device-loss handling and noexcept FFI chore(deps): update hwcodec to bbfc306 chore(qos): remove verbose diagnostics while retaining trace logs * update hwcodec Signed-off-by: 21pages <sunboeasy@gmail.com> * refactor(scrap): keep repeat encoding in the vram module * perf(vram): avoid intermediate frame vectors - Drain normal and repeated encode output into protobuf messages. - Cover null-texture encoding failures. - Clarify the count-based static refresh budget. - Update hwcodec to fix RAM decode buffers on resolution changes. Signed-off-by: 21pages <sunboeasy@gmail.com> * fix(scrap): preserve hardware encoder error codes * test(video): remove uptime dependency from static refresh tests * feat(video): configure static repeat limits by codec and quality * fix(video): exempt VPx and AV1 from static repeat failure limit * fix(video): count only successful static repeat frames * fix(video): refine static repeat scheduling and VPx frame handling Allow static refinement after a VP8/VP9 captured frame produces no output without enabling it after a genuine encoding error. Carry this distinction alongside the send result so the latest cached frame can still be sent when capture becomes idle. Keep VP8/VP9 repeats at a minimum of 100ms. Other codecs wait at least 100ms initially, then follow the QoS frame interval aligned with the capture loop. Use four quality bands, with VP8/VP9 limits of 30/20/20/0 and the tuned H264/H265 limits. Include only production behavior and regression tests. Validation: 16 static_refresh tests passed with hwcodec,vram,flutter enabled, covering latest-frame recovery, genuine errors, timing, and repeat budgets. * fix(video): tune static repeat limits by codec and backend Use the measured obvious-improvement checkpoints for VP8/VP9 and separate NVENC, QSV, and AMF H.264/HEVC limits for each quality tier. The checkpoints recover 80% of the observed error improvement across the tested inputs. Resolve the hardware backend from EncoderCfg during StaticRefresh construction, including the VRAM adapter vendor. Keep the existing limits for other hardware backends. Validation: 19 static_refresh tests passed with hwcodec,vram,flutter. The repeat measurements cover 192 series with no encode/decode errors. * fix(video): refine static refresh budgets and cap repeat FPS Add VideoToolbox HEVC budgets and tune quality bands from repeat-frame comparisons. Cap static refresh at 30 FPS and use the speed preset consistently at quality 0.5. * remove unused variables Signed-off-by: 21pages <sunboeasy@gmail.com> * fix(video): repair static refresh tests and throttle repeat logs --------- Signed-off-by: 21pages <sunboeasy@gmail.com>
This commit is contained in:
21pages
Generated
+2
-2
@@ -3888,8 +3888,8 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
|
||||
|
||||
[[package]]
|
||||
name = "hwcodec"
|
||||
version = "0.7.1"
|
||||
source = "git+https://github.com/rustdesk-org/hwcodec#778df1f99597722473b29443bac22ae6c23946fe"
|
||||
version = "0.8.0"
|
||||
source = "git+https://github.com/rustdesk-org/hwcodec#febec27e1b40bb9fbbae02e9c59b71ac824b9fe1"
|
||||
dependencies = [
|
||||
"bindgen 0.59.2",
|
||||
"cc",
|
||||
|
||||
@@ -25,6 +25,9 @@ use hwcodec::{
|
||||
},
|
||||
};
|
||||
|
||||
#[cfg(all(test, windows, feature = "vram"))]
|
||||
mod tests;
|
||||
|
||||
const DEFAULT_PIXFMT: AVPixelFormat = AVPixelFormat::AV_PIX_FMT_NV12;
|
||||
pub const DEFAULT_FPS: i32 = 30;
|
||||
const DEFAULT_GOP: i32 = i32::MAX;
|
||||
@@ -236,7 +239,7 @@ impl HwRamEncoder {
|
||||
data.append(v);
|
||||
Ok(data)
|
||||
}
|
||||
Err(_) => Ok(Vec::<EncodeFrame>::new()),
|
||||
Err(code) => Err(anyhow!("RAM encode failed: {}", code)),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
use super::{CodecInfo, HwCodecConfig, CONFIG, CONFIG_SET_BY_IPC};
|
||||
use crate::{
|
||||
codec::enable_vram_option,
|
||||
vram::{VRamDecoder, VRamEncoder},
|
||||
AdapterDevice, CodecFormat,
|
||||
};
|
||||
use base::config::keys::{OPTION_ALLOW_D3D_RENDER, OPTION_ENABLE_HWCODEC};
|
||||
use hbb_common::config::{OVERWRITE_LOCAL_SETTINGS, OVERWRITE_SETTINGS};
|
||||
use hwcodec::{
|
||||
common::{DataFormat, Driver},
|
||||
vram::{DecodeContext, FeatureContext},
|
||||
};
|
||||
use std::{collections::HashMap, mem};
|
||||
|
||||
struct RestoreConfig {
|
||||
config: Option<HwCodecConfig>,
|
||||
set_by_ipc: bool,
|
||||
options: HashMap<String, String>,
|
||||
local_options: HashMap<String, String>,
|
||||
}
|
||||
|
||||
impl Drop for RestoreConfig {
|
||||
fn drop(&mut self) {
|
||||
*CONFIG.lock().unwrap() = self.config.take();
|
||||
*CONFIG_SET_BY_IPC.lock().unwrap() = self.set_by_ipc;
|
||||
*OVERWRITE_SETTINGS.write().unwrap() = mem::take(&mut self.options);
|
||||
*OVERWRITE_LOCAL_SETTINGS.write().unwrap() = mem::take(&mut self.local_options);
|
||||
}
|
||||
}
|
||||
|
||||
fn add_backend(config: &mut HwCodecConfig, driver: Driver, vendor: Driver, format: DataFormat) {
|
||||
let luid = match vendor {
|
||||
Driver::NV => 1,
|
||||
Driver::MFX => 2,
|
||||
Driver::AMF => 3,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
config.vram_encode.push(FeatureContext {
|
||||
driver: driver.clone(),
|
||||
vendor: vendor.clone(),
|
||||
luid,
|
||||
data_format: format,
|
||||
});
|
||||
config.vram_decode.push(DecodeContext {
|
||||
device: None,
|
||||
driver,
|
||||
vendor,
|
||||
luid,
|
||||
data_format: format,
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cached_sdk_entries_are_ignored_before_check_completes() {
|
||||
let _restore = RestoreConfig {
|
||||
config: CONFIG.lock().unwrap().take(),
|
||||
set_by_ipc: mem::replace(&mut *CONFIG_SET_BY_IPC.lock().unwrap(), false),
|
||||
options: mem::take(&mut *OVERWRITE_SETTINGS.write().unwrap()),
|
||||
local_options: mem::take(&mut *OVERWRITE_LOCAL_SETTINGS.write().unwrap()),
|
||||
};
|
||||
OVERWRITE_SETTINGS
|
||||
.write()
|
||||
.unwrap()
|
||||
.insert(OPTION_ENABLE_HWCODEC.into(), "Y".into());
|
||||
OVERWRITE_LOCAL_SETTINGS
|
||||
.write()
|
||||
.unwrap()
|
||||
.insert(OPTION_ALLOW_D3D_RENDER.into(), "Y".into());
|
||||
assert!(enable_vram_option(false));
|
||||
|
||||
let formats = [
|
||||
(CodecFormat::H264, DataFormat::H264),
|
||||
(CodecFormat::H265, DataFormat::H265),
|
||||
];
|
||||
let mut cached = HwCodecConfig::default();
|
||||
for (_, format) in formats {
|
||||
// A RAM fallback keeps this selection test independent of attached displays.
|
||||
cached.ram_encode.push(CodecInfo {
|
||||
format,
|
||||
..Default::default()
|
||||
});
|
||||
for vendor in [Driver::NV, Driver::MFX, Driver::AMF] {
|
||||
add_backend(&mut cached, vendor.clone(), vendor, format);
|
||||
}
|
||||
}
|
||||
*CONFIG.lock().unwrap() = Some(cached.clone());
|
||||
assert!(!HwCodecConfig::already_set());
|
||||
for (format, _) in formats {
|
||||
assert!(VRamEncoder::available(format).is_empty());
|
||||
for luid in 1..=3 {
|
||||
assert!(VRamDecoder::try_get(format, Some(luid)).is_none());
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
VRamDecoder::possible_available_without_check(),
|
||||
(false, false)
|
||||
);
|
||||
|
||||
for (index, &(format, data_format)) in formats.iter().enumerate() {
|
||||
for vendor in [Driver::NV, Driver::MFX, Driver::AMF] {
|
||||
add_backend(&mut cached, Driver::FFMPEG, vendor, data_format);
|
||||
}
|
||||
*CONFIG.lock().unwrap() = Some(cached.clone());
|
||||
let encoders = VRamEncoder::available(format);
|
||||
assert_eq!(encoders.len(), 3);
|
||||
for encoder in encoders {
|
||||
assert_eq!(encoder.driver, Driver::FFMPEG);
|
||||
let device = AdapterDevice {
|
||||
luid: encoder.luid,
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(VRamEncoder::try_get(&device, format), Some(encoder.clone()));
|
||||
let decoder = VRamDecoder::try_get(format, Some(encoder.luid)).unwrap();
|
||||
assert_eq!(decoder.driver, Driver::FFMPEG);
|
||||
assert_eq!(decoder.vendor, encoder.vendor);
|
||||
assert_eq!(decoder.data_format, data_format);
|
||||
}
|
||||
assert!(VRamDecoder::available(format, Some(99)).is_empty());
|
||||
assert!(VRamDecoder::available(format, None).is_empty());
|
||||
assert_eq!(
|
||||
VRamDecoder::possible_available_without_check(),
|
||||
(true, index == 1)
|
||||
);
|
||||
}
|
||||
assert!(!HwCodecConfig::already_set());
|
||||
assert_eq!(
|
||||
HwCodecConfig::get().vram_encode.len(),
|
||||
cached.vram_encode.len()
|
||||
);
|
||||
}
|
||||
@@ -209,6 +209,8 @@ impl Frame<'_> {
|
||||
pub enum EncodeInput<'a> {
|
||||
YUV(&'a [u8]),
|
||||
Texture((*mut c_void, usize)),
|
||||
#[cfg(all(windows, feature = "vram"))]
|
||||
Repeat,
|
||||
}
|
||||
|
||||
impl<'a> EncodeInput<'a> {
|
||||
|
||||
@@ -20,6 +20,17 @@ use std::{ptr, slice};
|
||||
generate_call_macro!(call_vpx, false);
|
||||
generate_call_ptr_macro!(call_vpx_ptr);
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct VpxNoOutput;
|
||||
|
||||
impl std::fmt::Display for VpxNoOutput {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str("no valid frame")
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for VpxNoOutput {}
|
||||
|
||||
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
|
||||
pub enum VpxVideoCodecId {
|
||||
VP8,
|
||||
@@ -187,7 +198,7 @@ impl EncoderApi for VpxEncoder {
|
||||
if frames.len() > 0 {
|
||||
Ok(VpxEncoder::create_video_frame(self.id, frames))
|
||||
} else {
|
||||
Err(anyhow!("no valid frame"))
|
||||
Err(VpxNoOutput.into())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -15,8 +15,9 @@ use hbb_common::{
|
||||
bytes::Bytes,
|
||||
log, ResultType,
|
||||
};
|
||||
pub use hwcodec::common::Driver;
|
||||
use hwcodec::{
|
||||
common::{DataFormat, Driver, MAX_GOP},
|
||||
common::{DataFormat, MAX_GOP},
|
||||
vram::{
|
||||
decode::{self, DecodeFrame, Decoder},
|
||||
encode::{self, EncodeFrame, Encoder},
|
||||
@@ -24,6 +25,9 @@ use hwcodec::{
|
||||
},
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
mod repeat_tests;
|
||||
|
||||
// https://www.reddit.com/r/buildapc/comments/d2m4ny/two_graphics_cards_two_monitors/
|
||||
// https://www.reddit.com/r/techsupport/comments/t2v9u6/dual_monitor_setup_with_dual_gpu/
|
||||
// https://cybersided.com/two-monitors-two-gpus/
|
||||
@@ -48,8 +52,6 @@ pub struct VRamEncoder {
|
||||
pub format: DataFormat,
|
||||
ctx: EncodeContext,
|
||||
bitrate: u32,
|
||||
last_frame_len: usize,
|
||||
same_bad_len_counter: usize,
|
||||
}
|
||||
|
||||
impl EncoderApi for VRamEncoder {
|
||||
@@ -83,8 +85,6 @@ impl EncoderApi for VRamEncoder {
|
||||
ctx,
|
||||
format: config.feature.data_format,
|
||||
bitrate,
|
||||
last_frame_len: 0,
|
||||
same_bad_len_counter: 0,
|
||||
}),
|
||||
Err(_) => Err(anyhow!(format!("Failed to create encoder"))),
|
||||
}
|
||||
@@ -98,7 +98,15 @@ impl EncoderApi for VRamEncoder {
|
||||
frame: EncodeInput,
|
||||
ms: i64,
|
||||
) -> ResultType<base::message_proto::VideoFrame> {
|
||||
let (texture, rotation) = frame.texture()?;
|
||||
#[cfg(all(windows, feature = "vram"))]
|
||||
let repeated = matches!(&frame, EncodeInput::Repeat);
|
||||
#[cfg(not(all(windows, feature = "vram")))]
|
||||
let repeated = false;
|
||||
let (texture, rotation) = if repeated {
|
||||
(std::ptr::null_mut(), 0)
|
||||
} else {
|
||||
frame.texture()?
|
||||
};
|
||||
if rotation != 0 {
|
||||
// to-do: support rotation
|
||||
// Both the encoder and display(w,h) information need to be changed.
|
||||
@@ -106,10 +114,12 @@ impl EncoderApi for VRamEncoder {
|
||||
}
|
||||
let mut vf = VideoFrame::new();
|
||||
let mut frames = Vec::new();
|
||||
for frame in self
|
||||
.encode(texture, ms)
|
||||
.with_context(|| "Failed to encode")?
|
||||
{
|
||||
let result = if repeated {
|
||||
self.encode_repeat(ms)
|
||||
} else {
|
||||
self.encode(texture, ms)
|
||||
};
|
||||
for frame in result.with_context(|| "Failed to encode")? {
|
||||
frames.push(EncodedVideoFrame {
|
||||
data: Bytes::from(frame.data),
|
||||
pts: frame.pts,
|
||||
@@ -117,29 +127,10 @@ impl EncoderApi for VRamEncoder {
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
if repeated && frames.is_empty() {
|
||||
return Ok(vf);
|
||||
}
|
||||
if frames.len() > 0 {
|
||||
// This kind of problem is occurred after a period of time when using the AMD SDK,
|
||||
// the encoding length is fixed at about 40, and the picture is still
|
||||
const MIN_BAD_LEN: usize = 100;
|
||||
const MAX_BAD_COUNTER: usize = 30;
|
||||
let this_frame_len = frames[0].data.len();
|
||||
if self.ctx.f.driver == Driver::AMF
|
||||
&& this_frame_len < MIN_BAD_LEN
|
||||
&& this_frame_len == self.last_frame_len
|
||||
{
|
||||
self.same_bad_len_counter += 1;
|
||||
if self.same_bad_len_counter >= MAX_BAD_COUNTER {
|
||||
log::info!(
|
||||
"{} times encoding len is {}, switch",
|
||||
self.same_bad_len_counter,
|
||||
self.last_frame_len
|
||||
);
|
||||
bail!(crate::codec::ENCODE_NEED_SWITCH);
|
||||
}
|
||||
} else {
|
||||
self.same_bad_len_counter = 0;
|
||||
}
|
||||
self.last_frame_len = this_frame_len;
|
||||
let frames = EncodedVideoFrames {
|
||||
frames: frames.into(),
|
||||
..Default::default()
|
||||
@@ -244,6 +235,7 @@ impl VRamEncoder {
|
||||
let v: Vec<_> = crate::hwcodec::HwCodecConfig::get()
|
||||
.vram_encode
|
||||
.drain(..)
|
||||
.filter(|c| c.driver == Driver::FFMPEG)
|
||||
.filter(|c| c.data_format == data_format)
|
||||
.collect();
|
||||
if crate::hwcodec::HwRamEncoder::try_get(format).is_some() {
|
||||
@@ -274,14 +266,21 @@ impl VRamEncoder {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn encode(&mut self, texture: *mut c_void, ms: i64) -> ResultType<Vec<EncodeFrame>> {
|
||||
pub fn encode(
|
||||
&mut self,
|
||||
texture: *mut c_void,
|
||||
ms: i64,
|
||||
) -> ResultType<std::vec::Drain<'_, EncodeFrame>> {
|
||||
match self.encoder.encode(texture, ms) {
|
||||
Ok(v) => {
|
||||
let mut data = Vec::<EncodeFrame>::new();
|
||||
data.append(v);
|
||||
Ok(data)
|
||||
}
|
||||
Err(_) => Ok(Vec::<EncodeFrame>::new()),
|
||||
Ok(frames) => Ok(frames.drain(..)),
|
||||
Err(code) => Err(anyhow!("VRAM encode failed: {}", code)),
|
||||
}
|
||||
}
|
||||
|
||||
fn encode_repeat(&mut self, ms: i64) -> ResultType<std::vec::Drain<'_, EncodeFrame>> {
|
||||
match self.encoder.encode_repeat(ms) {
|
||||
Ok(frames) => Ok(frames.drain(..)),
|
||||
Err(code) => Err(anyhow!("VRAM repeat encode failed: {}", code)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -340,6 +339,7 @@ impl VRamDecoder {
|
||||
crate::hwcodec::HwCodecConfig::get()
|
||||
.vram_decode
|
||||
.drain(..)
|
||||
.filter(|c| c.driver == Driver::FFMPEG)
|
||||
.filter(|c| c.data_format == data_format && c.luid == luid && luid != 0)
|
||||
.collect()
|
||||
}
|
||||
@@ -350,8 +350,10 @@ impl VRamDecoder {
|
||||
}
|
||||
let v = crate::hwcodec::HwCodecConfig::get().vram_decode;
|
||||
(
|
||||
v.iter().any(|d| d.data_format == DataFormat::H264),
|
||||
v.iter().any(|d| d.data_format == DataFormat::H265),
|
||||
v.iter()
|
||||
.any(|d| d.driver == Driver::FFMPEG && d.data_format == DataFormat::H264),
|
||||
v.iter()
|
||||
.any(|d| d.driver == Driver::FFMPEG && d.data_format == DataFormat::H265),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,244 @@
|
||||
use super::VRamEncoder;
|
||||
use crate::{codec::EncoderApi, EncodeInput};
|
||||
use base::message_proto::{video_frame, VideoFrame};
|
||||
use hwcodec::{
|
||||
common::{DataFormat, Driver, MAX_GOP},
|
||||
ffmpeg::AVHWDeviceType,
|
||||
ffmpeg_ram::decode::{DecodeContext, Decoder},
|
||||
vram::{encode::Encoder, DynamicContext, EncodeContext, FeatureContext},
|
||||
};
|
||||
use std::{mem, ptr};
|
||||
use winapi::{
|
||||
shared::{
|
||||
dxgi::{CreateDXGIFactory1, IDXGIFactory1},
|
||||
dxgiformat::DXGI_FORMAT_B8G8R8A8_UNORM,
|
||||
dxgitype::DXGI_SAMPLE_DESC,
|
||||
winerror::DXGI_ERROR_NOT_FOUND,
|
||||
},
|
||||
um::{d3d11::*, d3dcommon::D3D_DRIVER_TYPE_UNKNOWN, unknwnbase::IUnknown},
|
||||
Interface,
|
||||
};
|
||||
|
||||
struct Com<T>(*mut T);
|
||||
|
||||
impl<T> Drop for Com<T> {
|
||||
fn drop(&mut self) {
|
||||
if !self.0.is_null() {
|
||||
unsafe { (*(self.0 as *mut IUnknown)).Release() };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn texture(
|
||||
device: *mut ID3D11Device,
|
||||
width: u32,
|
||||
height: u32,
|
||||
shade: u8,
|
||||
) -> Com<ID3D11Texture2D> {
|
||||
let pixels = pixels(width, height, shade);
|
||||
let desc = D3D11_TEXTURE2D_DESC {
|
||||
Width: width,
|
||||
Height: height,
|
||||
MipLevels: 1,
|
||||
ArraySize: 1,
|
||||
Format: DXGI_FORMAT_B8G8R8A8_UNORM,
|
||||
SampleDesc: DXGI_SAMPLE_DESC {
|
||||
Count: 1,
|
||||
Quality: 0,
|
||||
},
|
||||
Usage: D3D11_USAGE_DEFAULT,
|
||||
BindFlags: D3D11_BIND_SHADER_RESOURCE | D3D11_BIND_RENDER_TARGET,
|
||||
CPUAccessFlags: 0,
|
||||
MiscFlags: D3D11_RESOURCE_MISC_SHARED,
|
||||
};
|
||||
let data = D3D11_SUBRESOURCE_DATA {
|
||||
pSysMem: pixels.as_ptr() as _,
|
||||
SysMemPitch: width * 4,
|
||||
SysMemSlicePitch: 0,
|
||||
};
|
||||
let mut raw = ptr::null_mut();
|
||||
assert_eq!((*device).CreateTexture2D(&desc, &data, &mut raw), 0);
|
||||
Com(raw)
|
||||
}
|
||||
|
||||
fn pixels(width: u32, height: u32, shade: u8) -> Vec<u8> {
|
||||
[shade, shade, shade, 255].repeat((width * height) as usize)
|
||||
}
|
||||
|
||||
fn check_picture(
|
||||
decoder: &mut Decoder,
|
||||
frame: VideoFrame,
|
||||
width: u32,
|
||||
height: u32,
|
||||
bright: bool,
|
||||
pts: i64,
|
||||
) {
|
||||
let frames = match frame.union.unwrap() {
|
||||
video_frame::Union::H264s(frames) | video_frame::Union::H265s(frames) => frames,
|
||||
_ => panic!("unexpected codec"),
|
||||
};
|
||||
let mut decoded = 0;
|
||||
for packet in frames.frames {
|
||||
assert_eq!(packet.pts, pts);
|
||||
for picture in decoder.decode(&packet.data).unwrap() {
|
||||
assert_eq!((picture.width, picture.height), (width as _, height as _));
|
||||
let center = (height as usize / 2) * picture.linesize[0] as usize + width as usize / 2;
|
||||
let y = picture.data[0][center];
|
||||
assert!(
|
||||
if bright { y > 160 } else { y < 90 },
|
||||
"stale or overwritten pixels: expected bright={}, got Y={}",
|
||||
bright,
|
||||
y
|
||||
);
|
||||
decoded += 1;
|
||||
}
|
||||
}
|
||||
assert!(decoded > 0, "encoder output did not decode to a picture");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires hardware H.264/H.265 encoders; run explicitly on GPU hosts"]
|
||||
fn hardware_repeats_own_pixels_and_reset_on_recreation() {
|
||||
unsafe {
|
||||
let mut raw_factory = ptr::null_mut();
|
||||
assert_eq!(
|
||||
CreateDXGIFactory1(&IDXGIFactory1::uuidof(), &mut raw_factory),
|
||||
0
|
||||
);
|
||||
let factory = Com(raw_factory as *mut IDXGIFactory1);
|
||||
let mut tested = 0;
|
||||
for index in 0.. {
|
||||
let mut raw_adapter = ptr::null_mut();
|
||||
let result = (*factory.0).EnumAdapters1(index, &mut raw_adapter);
|
||||
if result == DXGI_ERROR_NOT_FOUND {
|
||||
break;
|
||||
}
|
||||
assert_eq!(result, 0);
|
||||
let adapter = Com(raw_adapter);
|
||||
let mut desc = mem::zeroed();
|
||||
assert_eq!((*adapter.0).GetDesc1(&mut desc), 0);
|
||||
let vendor = match desc.VendorId {
|
||||
0x10de => Driver::NV,
|
||||
0x1002 => Driver::AMF,
|
||||
0x8086 => Driver::MFX,
|
||||
_ => continue,
|
||||
};
|
||||
let luid = ((desc.AdapterLuid.HighPart as i64) << 32) | desc.AdapterLuid.LowPart as i64;
|
||||
let mut raw_device = ptr::null_mut();
|
||||
let mut raw_context = ptr::null_mut();
|
||||
assert_eq!(
|
||||
D3D11CreateDevice(
|
||||
adapter.0 as _,
|
||||
D3D_DRIVER_TYPE_UNKNOWN,
|
||||
ptr::null_mut(),
|
||||
D3D11_CREATE_DEVICE_BGRA_SUPPORT | D3D11_CREATE_DEVICE_VIDEO_SUPPORT,
|
||||
ptr::null(),
|
||||
0,
|
||||
D3D11_SDK_VERSION,
|
||||
&mut raw_device,
|
||||
ptr::null_mut(),
|
||||
&mut raw_context,
|
||||
),
|
||||
0
|
||||
);
|
||||
let device = Com(raw_device);
|
||||
let context = Com(raw_context);
|
||||
for driver in [Driver::FFMPEG, vendor.clone()] {
|
||||
for format in [DataFormat::H264, DataFormat::H265] {
|
||||
for (width, height) in [(320, 240), (640, 360)] {
|
||||
let ctx = EncodeContext {
|
||||
f: FeatureContext {
|
||||
driver: driver.clone(),
|
||||
vendor: vendor.clone(),
|
||||
luid,
|
||||
data_format: format,
|
||||
},
|
||||
d: DynamicContext {
|
||||
device: Some(device.0 as _),
|
||||
width: width as _,
|
||||
height: height as _,
|
||||
kbitrate: 1000,
|
||||
framerate: 30,
|
||||
gop: MAX_GOP as _,
|
||||
},
|
||||
};
|
||||
let native = match Encoder::new(ctx.clone()) {
|
||||
Ok(encoder) => encoder,
|
||||
Err(()) => {
|
||||
eprintln!(
|
||||
"SKIP unavailable {:?} {:?} {:?} adapter={luid:#x}",
|
||||
vendor, driver, format
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let mut encoder = VRamEncoder {
|
||||
encoder: native,
|
||||
format,
|
||||
ctx,
|
||||
bitrate: 1000,
|
||||
};
|
||||
assert!(encoder.encode_to_message(EncodeInput::Repeat, 0).is_err());
|
||||
assert!(encoder
|
||||
.encode_to_message(EncodeInput::Texture((ptr::null_mut(), 0)), 0)
|
||||
.is_err());
|
||||
let mut decoder = Decoder::new(DecodeContext {
|
||||
name: if format == DataFormat::H264 {
|
||||
"h264"
|
||||
} else {
|
||||
"hevc"
|
||||
}
|
||||
.into(),
|
||||
device_type: AVHWDeviceType::AV_HWDEVICE_TYPE_NONE,
|
||||
thread_count: 1,
|
||||
})
|
||||
.unwrap();
|
||||
let source = texture(device.0, width, height, 32);
|
||||
let frame = encoder
|
||||
.encode_to_message(EncodeInput::Texture((source.0 as _, 0)), 0)
|
||||
.unwrap();
|
||||
check_picture(&mut decoder, frame, width, height, false, 0);
|
||||
|
||||
// Overwrite and release the capture resource before any repeats.
|
||||
let overwritten = pixels(width, height, 224);
|
||||
(*context.0).UpdateSubresource(
|
||||
source.0 as _,
|
||||
0,
|
||||
ptr::null(),
|
||||
overwritten.as_ptr() as _,
|
||||
width * 4,
|
||||
0,
|
||||
);
|
||||
(*context.0).Flush();
|
||||
drop(source);
|
||||
for count in 1..=100 {
|
||||
let pts = count * 100;
|
||||
let frame =
|
||||
encoder.encode_to_message(EncodeInput::Repeat, pts).unwrap();
|
||||
check_picture(&mut decoder, frame, width, height, false, pts);
|
||||
}
|
||||
|
||||
let source = texture(device.0, width, height, 224);
|
||||
let frame = encoder
|
||||
.encode_to_message(EncodeInput::Texture((source.0 as _, 0)), 10100)
|
||||
.unwrap();
|
||||
check_picture(&mut decoder, frame, width, height, true, 10100);
|
||||
drop(source);
|
||||
encoder.encoder.set_bitrate(1500).unwrap();
|
||||
let frame = encoder
|
||||
.encode_to_message(EncodeInput::Repeat, 10200)
|
||||
.unwrap();
|
||||
check_picture(&mut decoder, frame, width, height, true, 10200);
|
||||
eprintln!(
|
||||
"PASS {:?} {:?} {:?} {width}x{height} adapter={luid:#x}",
|
||||
vendor, driver, format
|
||||
);
|
||||
tested += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(tested > 0, "no hardware encoders were tested");
|
||||
eprintln!("Validated {tested} encoder/codec/resolution combinations");
|
||||
}
|
||||
}
|
||||
@@ -64,6 +64,8 @@ use std::{
|
||||
|
||||
pub const OPTION_REFRESH: &'static str = "refresh";
|
||||
|
||||
mod static_refresh;
|
||||
|
||||
#[cfg(windows)]
|
||||
const DXGI_RECOVERY_LIMIT: usize = 3;
|
||||
#[cfg(windows)]
|
||||
@@ -731,6 +733,16 @@ fn run(vs: VideoService) -> ResultType<()> {
|
||||
let mut encoder_warmed_up = false;
|
||||
let capture_width = c.width;
|
||||
let capture_height = c.height;
|
||||
let mut static_refresh = static_refresh::StaticRefresh::new(
|
||||
vs.source,
|
||||
codec_format,
|
||||
&encoder_cfg,
|
||||
&sp,
|
||||
&recorder,
|
||||
display_idx,
|
||||
capture_width,
|
||||
capture_height,
|
||||
);
|
||||
let (mut second_instant, mut send_counter) = (Instant::now(), 0);
|
||||
|
||||
while sp.ok() {
|
||||
@@ -855,7 +867,8 @@ fn run(vs: VideoService) -> ResultType<()> {
|
||||
}
|
||||
|
||||
let frame = frame.to(encoder.yuvfmt(), &mut yuv, &mut mid_data)?;
|
||||
let send_conn_ids = handle_one_frame(
|
||||
static_refresh.on_frame(&frame);
|
||||
let result = handle_one_frame(
|
||||
display_idx,
|
||||
&sp,
|
||||
frame,
|
||||
@@ -868,7 +881,8 @@ fn run(vs: VideoService) -> ResultType<()> {
|
||||
capture_width,
|
||||
capture_height,
|
||||
)?;
|
||||
frame_controller.set_send(now, send_conn_ids);
|
||||
static_refresh.on_frame_encoded(&result);
|
||||
frame_controller.set_send(now, result.send_conn_ids);
|
||||
send_counter += 1;
|
||||
}
|
||||
#[cfg(windows)]
|
||||
@@ -932,11 +946,22 @@ fn run(vs: VideoService) -> ResultType<()> {
|
||||
&mut encoder_warmed_up,
|
||||
capture_width,
|
||||
capture_height,
|
||||
)?;
|
||||
)?
|
||||
.send_conn_ids;
|
||||
static_refresh.on_encoded(!send_conn_ids.is_empty());
|
||||
frame_controller.set_send(now, send_conn_ids);
|
||||
send_counter += 1;
|
||||
}
|
||||
}
|
||||
static_refresh.try_encode(
|
||||
&yuv,
|
||||
spf,
|
||||
quality,
|
||||
now,
|
||||
ms,
|
||||
&mut encoder,
|
||||
&mut frame_controller,
|
||||
)?;
|
||||
}
|
||||
Err(err) => {
|
||||
#[cfg(windows)]
|
||||
@@ -1261,7 +1286,7 @@ fn handle_one_frame(
|
||||
encoder_warmed_up: &mut bool,
|
||||
width: usize,
|
||||
height: usize,
|
||||
) -> ResultType<HashSet<i32>> {
|
||||
) -> ResultType<static_refresh::FrameEncodeResult> {
|
||||
sp.snapshot(|sps| {
|
||||
// so that new sub and old sub share the same encoder after switch
|
||||
if sps.has_subscribes() {
|
||||
@@ -1272,6 +1297,7 @@ fn handle_one_frame(
|
||||
})?;
|
||||
|
||||
let mut send_conn_ids: HashSet<i32> = Default::default();
|
||||
let mut vpx_no_output = false;
|
||||
let first = *first_frame;
|
||||
*first_frame = false;
|
||||
match encoder.encode_to_message(frame, ms) {
|
||||
@@ -1289,6 +1315,7 @@ fn handle_one_frame(
|
||||
send_conn_ids = sp.send_video_frame(msg);
|
||||
}
|
||||
Err(e) => {
|
||||
vpx_no_output = e.is::<scrap::VpxNoOutput>();
|
||||
*encode_fail_counter += 1;
|
||||
// Encoding errors are not frequent except on Android
|
||||
if !cfg!(target_os = "android") {
|
||||
@@ -1319,7 +1346,10 @@ fn handle_one_frame(
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(send_conn_ids)
|
||||
Ok(static_refresh::FrameEncodeResult {
|
||||
send_conn_ids,
|
||||
vpx_no_output,
|
||||
})
|
||||
}
|
||||
|
||||
#[inline]
|
||||
|
||||
@@ -0,0 +1,745 @@
|
||||
use super::{GenericService, VideoFrameController, VideoSource};
|
||||
use base::message_proto::Message;
|
||||
use hbb_common::{bail, ResultType};
|
||||
use scrap::{
|
||||
codec::{Encoder, EncoderCfg, BR_BALANCED, BR_BEST, BR_SPEED},
|
||||
record::Recorder,
|
||||
CodecFormat, EncodeInput,
|
||||
};
|
||||
use std::{
|
||||
collections::HashSet,
|
||||
sync::{Arc, Mutex},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
const INITIAL_REPEAT_INTERVAL: Duration = Duration::from_millis(100);
|
||||
const MAX_REPEAT_FPS: u32 = 30;
|
||||
const MAX_REPEAT_FAILURES: usize = 3;
|
||||
const MAX_REPEAT_NO_OUTPUTS: usize = 100;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum HardwareBackend {
|
||||
Nvenc,
|
||||
Qsv,
|
||||
Amf,
|
||||
Vaapi,
|
||||
VideoToolbox,
|
||||
Other,
|
||||
}
|
||||
|
||||
fn max_repeat_frames(codec: CodecFormat, quality: f32, backend: HardwareBackend) -> usize {
|
||||
use HardwareBackend::*;
|
||||
// Measured entries sample quality 0.1, BR_SPEED, BR_BALANCED, and BR_BEST.
|
||||
// For measured entries, Original values come from AI-run tests. Active values
|
||||
// were adjusted by comparing 1080p screenshots captured every 10 repeat outputs.
|
||||
let (low, low_to_balanced, balanced_to_best, best) = match (codec, backend) {
|
||||
(CodecFormat::VP8, _) => (20, 20, 20, 30), // Original: (20, 20, 20, 30)
|
||||
(CodecFormat::VP9, _) => (40, 20, 20, 20), // Original: (40, 20, 20, 20)
|
||||
(CodecFormat::H264, Nvenc) => (80, 50, 60, 30), // Original: (270, 50, 60, 30)
|
||||
(CodecFormat::H264, Qsv) => (80, 70, 130, 110), // Original: (80, 70, 130, 110)
|
||||
(CodecFormat::H264, Amf) => (150, 150, 170, 110), // Original: (560, 150, 170, 110)
|
||||
(CodecFormat::H264, Vaapi) => (150, 100, 90, 60), // Original: (330, 100, 90, 60)
|
||||
(CodecFormat::H265, Nvenc) => (160, 90, 60, 30), // Original: (160, 90, 60, 30)
|
||||
(CodecFormat::H265, Qsv) => (150, 30, 30, 30), // Original: (150, 30, 30, 30)
|
||||
(CodecFormat::H265, Amf) => (200, 200, 200, 120), // Original: (580, 200, 200, 120)
|
||||
// The capture loop already repeats ten frames to drain VideoToolbox latency.
|
||||
(CodecFormat::H265, VideoToolbox) => (90, 90, 50, 90), // Original: (230, 90, 50, 90)
|
||||
(CodecFormat::H264, _) => (90, 60, 60, 30),
|
||||
(CodecFormat::H265, _) => (90, 60, 60, 30),
|
||||
(CodecFormat::AV1 | CodecFormat::Unknown, _) => return 0,
|
||||
};
|
||||
if quality >= BR_BEST {
|
||||
best
|
||||
} else if quality >= BR_BALANCED {
|
||||
balanced_to_best
|
||||
} else if quality >= BR_SPEED {
|
||||
low_to_balanced
|
||||
} else {
|
||||
low
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(super) struct FrameEncodeResult {
|
||||
pub(super) send_conn_ids: HashSet<i32>,
|
||||
pub(super) vpx_no_output: bool,
|
||||
}
|
||||
|
||||
pub(super) struct StaticRefresh<'a> {
|
||||
source: VideoSource,
|
||||
codec_format: CodecFormat,
|
||||
hardware_backend: HardwareBackend,
|
||||
sp: &'a GenericService,
|
||||
recorder: &'a Arc<Mutex<Option<Recorder>>>,
|
||||
display_idx: usize,
|
||||
width: usize,
|
||||
height: usize,
|
||||
last_encode: Instant,
|
||||
#[cfg(test)]
|
||||
elapsed_since_encode: Duration,
|
||||
repeat_output_counter: usize,
|
||||
repeat_no_outputs: usize,
|
||||
repeat_failures: usize,
|
||||
source_ready: bool,
|
||||
#[cfg(all(windows, feature = "vram"))]
|
||||
has_texture: bool,
|
||||
}
|
||||
|
||||
impl<'a> StaticRefresh<'a> {
|
||||
pub(super) fn new(
|
||||
source: VideoSource,
|
||||
codec_format: CodecFormat,
|
||||
encoder_config: &EncoderCfg,
|
||||
sp: &'a GenericService,
|
||||
recorder: &'a Arc<Mutex<Option<Recorder>>>,
|
||||
display_idx: usize,
|
||||
width: usize,
|
||||
height: usize,
|
||||
) -> Self {
|
||||
let hardware_backend = match encoder_config {
|
||||
#[cfg(feature = "hwcodec")]
|
||||
EncoderCfg::HWRAM(config) => match config.name.as_str() {
|
||||
"h264_nvenc" | "hevc_nvenc" => HardwareBackend::Nvenc,
|
||||
"h264_qsv" | "hevc_qsv" => HardwareBackend::Qsv,
|
||||
"h264_amf" | "hevc_amf" => HardwareBackend::Amf,
|
||||
"h264_vaapi" => HardwareBackend::Vaapi,
|
||||
"hevc_videotoolbox" => HardwareBackend::VideoToolbox,
|
||||
_ => HardwareBackend::Other,
|
||||
},
|
||||
#[cfg(feature = "vram")]
|
||||
// The FFMPEG VRAM driver selects the encoder from the adapter vendor.
|
||||
EncoderCfg::VRAM(config) => match config.feature.vendor {
|
||||
scrap::vram::Driver::NV => HardwareBackend::Nvenc,
|
||||
scrap::vram::Driver::MFX => HardwareBackend::Qsv,
|
||||
scrap::vram::Driver::AMF => HardwareBackend::Amf,
|
||||
_ => HardwareBackend::Other,
|
||||
},
|
||||
_ => HardwareBackend::Other,
|
||||
};
|
||||
Self {
|
||||
source,
|
||||
codec_format,
|
||||
hardware_backend,
|
||||
sp,
|
||||
recorder,
|
||||
display_idx,
|
||||
width,
|
||||
height,
|
||||
last_encode: Instant::now(),
|
||||
#[cfg(test)]
|
||||
elapsed_since_encode: Duration::ZERO,
|
||||
repeat_output_counter: 0,
|
||||
repeat_no_outputs: 0,
|
||||
repeat_failures: 0,
|
||||
source_ready: false,
|
||||
#[cfg(all(windows, feature = "vram"))]
|
||||
has_texture: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn on_frame(&mut self, _frame: &EncodeInput) {
|
||||
self.repeat_output_counter = 0;
|
||||
self.repeat_no_outputs = 0;
|
||||
self.source_ready = false;
|
||||
#[cfg(all(windows, feature = "vram"))]
|
||||
{
|
||||
self.has_texture =
|
||||
matches!(_frame, EncodeInput::Texture((texture, 0)) if !texture.is_null());
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn on_frame_encoded(&mut self, result: &FrameEncodeResult) {
|
||||
// VP8/VP9 rate control may drop a frame without an encoding error.
|
||||
// The latest input is still cached, so allow repeats even without output;
|
||||
// otherwise the final update may never be sent once capture becomes idle.
|
||||
self.on_encoded(
|
||||
!result.send_conn_ids.is_empty()
|
||||
|| (matches!(self.codec_format, CodecFormat::VP8 | CodecFormat::VP9)
|
||||
&& result.vpx_no_output),
|
||||
);
|
||||
}
|
||||
|
||||
pub(super) fn on_encoded(&mut self, success: bool) {
|
||||
self.last_encode = Instant::now();
|
||||
#[cfg(test)]
|
||||
{
|
||||
self.elapsed_since_encode = Duration::ZERO;
|
||||
}
|
||||
if success {
|
||||
self.source_ready = true;
|
||||
self.repeat_failures = 0;
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn try_encode(
|
||||
&mut self,
|
||||
yuv: &[u8],
|
||||
spf: Duration,
|
||||
quality: f32,
|
||||
now: Instant,
|
||||
ms: i64,
|
||||
encoder: &mut Encoder,
|
||||
frame_controller: &mut VideoFrameController,
|
||||
) -> ResultType<()> {
|
||||
let spf = spf.max(Duration::from_secs(1) / MAX_REPEAT_FPS);
|
||||
if !self.source.is_monitor()
|
||||
|| self.codec_format == CodecFormat::AV1
|
||||
|| !self.source_ready
|
||||
|| (self.repeat_failures >= MAX_REPEAT_FAILURES
|
||||
&& !matches!(
|
||||
self.codec_format,
|
||||
CodecFormat::VP8 | CodecFormat::VP9 | CodecFormat::AV1
|
||||
))
|
||||
|| self.repeat_no_outputs >= MAX_REPEAT_NO_OUTPUTS
|
||||
|| self.repeat_output_counter
|
||||
>= max_repeat_frames(self.codec_format, quality, self.hardware_backend)
|
||||
|| {
|
||||
#[cfg(not(test))]
|
||||
let elapsed = self.last_encode.elapsed();
|
||||
#[cfg(test)]
|
||||
let elapsed = self.elapsed_since_encode;
|
||||
let interval = if matches!(self.codec_format, CodecFormat::VP8 | CodecFormat::VP9)
|
||||
|| (self.repeat_output_counter == 0 && self.repeat_no_outputs == 0)
|
||||
{
|
||||
INITIAL_REPEAT_INTERVAL.max(spf)
|
||||
} else {
|
||||
spf
|
||||
};
|
||||
elapsed < interval
|
||||
}
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
let frame = if yuv.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(EncodeInput::YUV(yuv))
|
||||
};
|
||||
#[cfg(all(windows, feature = "vram"))]
|
||||
let frame = frame.or_else(|| self.has_texture.then_some(EncodeInput::Repeat));
|
||||
if let Some(frame) = frame {
|
||||
self.sp.snapshot(|sps| {
|
||||
if sps.has_subscribes() {
|
||||
bail!("SWITCH");
|
||||
}
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
let result = encoder.encode_to_message(frame, ms);
|
||||
self.last_encode = if matches!(self.codec_format, CodecFormat::VP8 | CodecFormat::VP9) {
|
||||
Instant::now()
|
||||
} else {
|
||||
// Align with the capture loop so encoding time does not skip the next tick.
|
||||
now
|
||||
};
|
||||
#[cfg(test)]
|
||||
{
|
||||
self.elapsed_since_encode = Duration::ZERO;
|
||||
}
|
||||
let mut vf = match result {
|
||||
Ok(vf) if vf.union.is_none() => {
|
||||
self.repeat_no_outputs += 1;
|
||||
return Ok(());
|
||||
}
|
||||
Ok(vf) => vf,
|
||||
Err(error) => {
|
||||
self.repeat_no_outputs += 1;
|
||||
self.repeat_failures += 1;
|
||||
if error.is::<scrap::VpxNoOutput>() {
|
||||
hbb_common::throttled_log!(
|
||||
Duration::from_secs(60),
|
||||
trace,
|
||||
"static refresh produced no output: {error:?}"
|
||||
);
|
||||
} else {
|
||||
hbb_common::throttled_log!(
|
||||
Duration::from_secs(60),
|
||||
debug,
|
||||
"static refresh failed ({}): {error:?}",
|
||||
self.repeat_failures
|
||||
);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
self.repeat_output_counter += 1;
|
||||
self.repeat_failures = 0;
|
||||
vf.display = self.display_idx as _;
|
||||
let mut msg = Message::new();
|
||||
msg.set_video_frame(vf);
|
||||
self.recorder
|
||||
.lock()
|
||||
.unwrap()
|
||||
.as_mut()
|
||||
.map(|r| r.write_message(&msg, self.width, self.height));
|
||||
let send_conn_ids = self.sp.send_video_frame(msg);
|
||||
frame_controller.set_send(now, send_conn_ids);
|
||||
// Static refinement must not count as motion for adaptive bitrate.
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::server::{connection::ConnInner, service::Service};
|
||||
use base::message_proto::VideoFrame;
|
||||
use hbb_common::tokio::sync::mpsc;
|
||||
use scrap::{
|
||||
codec::{EncoderApi, EncoderCfg, BR_BALANCED},
|
||||
vpxcodec::{VpxEncoderConfig, VpxVideoCodecId},
|
||||
EncodeYuvFormat,
|
||||
};
|
||||
use std::{cell::Cell, rc::Rc};
|
||||
|
||||
pub(super) fn new_refresh<'a>(
|
||||
source: VideoSource,
|
||||
codec_format: CodecFormat,
|
||||
sp: &'a GenericService,
|
||||
recorder: &'a Arc<Mutex<Option<Recorder>>>,
|
||||
display_idx: usize,
|
||||
width: usize,
|
||||
height: usize,
|
||||
) -> StaticRefresh<'a> {
|
||||
// The mock encoders simulate each format without a hardware backend.
|
||||
let config = EncoderCfg::VPX(VpxEncoderConfig {
|
||||
width: width as _,
|
||||
height: height as _,
|
||||
quality: BR_BALANCED,
|
||||
codec: VpxVideoCodecId::VP9,
|
||||
keyframe_interval: None,
|
||||
});
|
||||
StaticRefresh::new(
|
||||
source,
|
||||
codec_format,
|
||||
&config,
|
||||
sp,
|
||||
recorder,
|
||||
display_idx,
|
||||
width,
|
||||
height,
|
||||
)
|
||||
}
|
||||
|
||||
struct CountingEncoder(Rc<Cell<usize>>);
|
||||
|
||||
impl EncoderApi for CountingEncoder {
|
||||
fn new(_: EncoderCfg, _: bool) -> ResultType<Self> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn encode_to_message(&mut self, frame: EncodeInput, _: i64) -> ResultType<VideoFrame> {
|
||||
match frame {
|
||||
EncodeInput::YUV(yuv) => assert_eq!(yuv, &[1]),
|
||||
#[cfg(all(windows, feature = "vram"))]
|
||||
EncodeInput::Repeat => {}
|
||||
_ => panic!("unexpected encoder input"),
|
||||
}
|
||||
self.0.set(self.0.get() + 1);
|
||||
let mut vf = VideoFrame::new();
|
||||
vf.set_h264s(Default::default());
|
||||
Ok(vf)
|
||||
}
|
||||
|
||||
fn yuvfmt(&self) -> EncodeYuvFormat {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
#[cfg(feature = "vram")]
|
||||
fn input_texture(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn set_quality(&mut self, _: f32) -> ResultType<()> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn bitrate(&self) -> u32 {
|
||||
0
|
||||
}
|
||||
|
||||
fn support_changing_quality(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn latency_free(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn is_hardware(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn disable(&self) {
|
||||
unreachable!()
|
||||
}
|
||||
}
|
||||
|
||||
fn attempt(
|
||||
refresh: &mut StaticRefresh<'_>,
|
||||
calls: &Rc<Cell<usize>>,
|
||||
yuv: &[u8],
|
||||
spf: Duration,
|
||||
) {
|
||||
let mut encoder = Encoder {
|
||||
codec: Box::new(CountingEncoder(Rc::clone(calls))),
|
||||
};
|
||||
refresh
|
||||
.try_encode(
|
||||
yuv,
|
||||
spf,
|
||||
BR_BALANCED,
|
||||
Instant::now(),
|
||||
0,
|
||||
&mut encoder,
|
||||
&mut VideoFrameController::new(0),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn long_network_waits_preserve_budget_and_new_frames_restart_it() {
|
||||
let sp = GenericService::new("static-refresh-test".to_owned(), false);
|
||||
let recorder = Arc::new(Mutex::new(None));
|
||||
let mut refresh = new_refresh(
|
||||
VideoSource::Monitor,
|
||||
CodecFormat::VP9,
|
||||
&sp,
|
||||
&recorder,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
);
|
||||
let calls = Rc::new(Cell::new(0));
|
||||
let spf = Duration::from_millis(100);
|
||||
refresh.on_frame(&EncodeInput::YUV(&[1]));
|
||||
refresh.on_encoded(true);
|
||||
let limit = max_repeat_frames(CodecFormat::VP9, BR_BALANCED, HardwareBackend::Other);
|
||||
for _ in 0..limit {
|
||||
refresh.elapsed_since_encode = Duration::from_secs(60);
|
||||
attempt(&mut refresh, &calls, &[1], spf);
|
||||
}
|
||||
assert_eq!(calls.get(), limit);
|
||||
|
||||
refresh.elapsed_since_encode = Duration::from_secs(600);
|
||||
attempt(&mut refresh, &calls, &[1], spf);
|
||||
assert_eq!(calls.get(), limit);
|
||||
|
||||
refresh.on_frame(&EncodeInput::YUV(&[1]));
|
||||
refresh.on_encoded(true);
|
||||
refresh.elapsed_since_encode = Duration::from_secs(1);
|
||||
attempt(&mut refresh, &calls, &[1], spf);
|
||||
assert_eq!(calls.get(), limit + 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn initial_delay_repeat_interval_and_qos_limit_refresh_rate() {
|
||||
let sp = GenericService::new("static-refresh-test".to_owned(), false);
|
||||
let recorder = Arc::new(Mutex::new(None));
|
||||
let mut refresh = new_refresh(
|
||||
VideoSource::Monitor,
|
||||
CodecFormat::H264,
|
||||
&sp,
|
||||
&recorder,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
);
|
||||
let calls = Rc::new(Cell::new(0));
|
||||
refresh.on_frame(&EncodeInput::YUV(&[1]));
|
||||
refresh.on_encoded(true);
|
||||
attempt(&mut refresh, &calls, &[1], Duration::from_millis(10));
|
||||
assert_eq!(calls.get(), 0);
|
||||
|
||||
refresh.elapsed_since_encode = Duration::from_millis(50);
|
||||
attempt(&mut refresh, &calls, &[1], Duration::from_millis(10));
|
||||
assert_eq!(calls.get(), 0);
|
||||
|
||||
refresh.elapsed_since_encode = Duration::from_millis(100);
|
||||
attempt(&mut refresh, &calls, &[1], Duration::from_millis(10));
|
||||
assert_eq!(calls.get(), 1);
|
||||
|
||||
refresh.elapsed_since_encode = Duration::from_millis(33);
|
||||
attempt(&mut refresh, &calls, &[1], Duration::from_millis(10));
|
||||
assert_eq!(calls.get(), 1);
|
||||
|
||||
refresh.elapsed_since_encode = Duration::from_secs(1) / 30;
|
||||
attempt(&mut refresh, &calls, &[1], Duration::from_millis(10));
|
||||
assert_eq!(calls.get(), 2);
|
||||
|
||||
refresh.elapsed_since_encode = Duration::from_millis(49);
|
||||
attempt(&mut refresh, &calls, &[1], Duration::from_millis(50));
|
||||
assert_eq!(calls.get(), 2);
|
||||
refresh.elapsed_since_encode = Duration::from_millis(50);
|
||||
attempt(&mut refresh, &calls, &[1], Duration::from_millis(50));
|
||||
assert_eq!(calls.get(), 3);
|
||||
|
||||
refresh.on_frame(&EncodeInput::YUV(&[1]));
|
||||
refresh.on_encoded(true);
|
||||
refresh.elapsed_since_encode = Duration::from_millis(50);
|
||||
attempt(&mut refresh, &calls, &[1], Duration::from_millis(10));
|
||||
assert_eq!(calls.get(), 3);
|
||||
|
||||
refresh.elapsed_since_encode = Duration::from_secs(1);
|
||||
attempt(&mut refresh, &calls, &[1], Duration::from_secs(2));
|
||||
assert_eq!(calls.get(), 3);
|
||||
|
||||
refresh.elapsed_since_encode = Duration::from_secs(3);
|
||||
attempt(&mut refresh, &calls, &[1], Duration::from_secs(2));
|
||||
assert_eq!(calls.get(), 4);
|
||||
attempt(&mut refresh, &calls, &[1], Duration::from_secs(2));
|
||||
assert_eq!(calls.get(), 4);
|
||||
refresh.elapsed_since_encode = Duration::from_millis(1999);
|
||||
attempt(&mut refresh, &calls, &[1], Duration::from_secs(2));
|
||||
assert_eq!(calls.get(), 4);
|
||||
refresh.elapsed_since_encode = Duration::from_millis(2000);
|
||||
attempt(&mut refresh, &calls, &[1], Duration::from_secs(2));
|
||||
assert_eq!(calls.get(), 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(all(windows, feature = "vram"))]
|
||||
fn texture_frames_request_repeats_without_accessing_capture_resources() {
|
||||
let sp = GenericService::new("static-refresh-test".to_owned(), false);
|
||||
let recorder = Arc::new(Mutex::new(None));
|
||||
let mut refresh = new_refresh(
|
||||
VideoSource::Monitor,
|
||||
CodecFormat::H264,
|
||||
&sp,
|
||||
&recorder,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
);
|
||||
let calls = Rc::new(Cell::new(0));
|
||||
// Deliberately not a D3D resource: neither hook may dereference it.
|
||||
let texture = std::ptr::NonNull::<u8>::dangling().as_ptr() as _;
|
||||
for _ in 0..60 {
|
||||
refresh.on_frame(&EncodeInput::Texture((texture, 0)));
|
||||
}
|
||||
refresh.on_encoded(true);
|
||||
refresh.elapsed_since_encode = Duration::from_secs(1);
|
||||
attempt(&mut refresh, &calls, &[], Duration::from_millis(16));
|
||||
assert_eq!(calls.get(), 1);
|
||||
|
||||
for frame in [
|
||||
EncodeInput::YUV(&[1]),
|
||||
EncodeInput::Texture((texture, 90)),
|
||||
EncodeInput::Texture((std::ptr::null_mut(), 0)),
|
||||
] {
|
||||
refresh.on_frame(&frame);
|
||||
refresh.on_encoded(true);
|
||||
refresh.elapsed_since_encode = Duration::from_secs(1);
|
||||
attempt(&mut refresh, &calls, &[], Duration::from_millis(16));
|
||||
assert_eq!(calls.get(), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cameras_av1_and_missing_frames_are_not_refreshed() {
|
||||
let sp = GenericService::new("static-refresh-test".to_owned(), false);
|
||||
let recorder = Arc::new(Mutex::new(None));
|
||||
let calls = Rc::new(Cell::new(0));
|
||||
for (source, codec_format, yuv) in [
|
||||
(VideoSource::Camera, CodecFormat::VP9, &[1][..]),
|
||||
(VideoSource::Monitor, CodecFormat::AV1, &[1][..]),
|
||||
(VideoSource::Monitor, CodecFormat::VP9, &[][..]),
|
||||
] {
|
||||
let mut refresh = new_refresh(source, codec_format, &sp, &recorder, 0, 1, 1);
|
||||
refresh.on_frame(&EncodeInput::YUV(yuv));
|
||||
refresh.on_encoded(true);
|
||||
refresh.elapsed_since_encode = Duration::from_secs(60);
|
||||
attempt(&mut refresh, &calls, yuv, Duration::from_millis(100));
|
||||
}
|
||||
assert_eq!(calls.get(), 0);
|
||||
}
|
||||
|
||||
fn assert_output_budget(
|
||||
refresh: &mut StaticRefresh<'_>,
|
||||
sp: &GenericService,
|
||||
quality: f32,
|
||||
expected: usize,
|
||||
) {
|
||||
let (tx, rx) = mpsc::unbounded_channel();
|
||||
sp.on_subscribe(ConnInner::new(1, None, Some(tx)));
|
||||
let mut encoder = Encoder {
|
||||
codec: Box::new(CountingEncoder(Rc::new(Cell::new(0)))),
|
||||
};
|
||||
refresh.on_frame(&EncodeInput::YUV(&[1]));
|
||||
refresh.on_encoded(true);
|
||||
for _ in 0..=expected {
|
||||
refresh.elapsed_since_encode = Duration::from_secs(1);
|
||||
refresh
|
||||
.try_encode(
|
||||
&[1],
|
||||
Duration::from_millis(33),
|
||||
quality,
|
||||
Instant::now(),
|
||||
0,
|
||||
&mut encoder,
|
||||
&mut VideoFrameController::new(0),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
assert_eq!(rx.len(), expected);
|
||||
sp.on_unsubscribe(1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quality_bands_stop_each_backend_at_its_output_budget() {
|
||||
let sp = GenericService::new("repeat-quality-test".to_owned(), false);
|
||||
let recorder = Arc::new(Mutex::new(None));
|
||||
assert_eq!(
|
||||
[0.1, BR_SPEED, BR_BALANCED, BR_BEST]
|
||||
.map(|q| { max_repeat_frames(CodecFormat::H264, q, HardwareBackend::Vaapi) }),
|
||||
[150, 100, 90, 60]
|
||||
);
|
||||
for codec in [
|
||||
CodecFormat::VP8,
|
||||
CodecFormat::VP9,
|
||||
CodecFormat::H264,
|
||||
CodecFormat::H265,
|
||||
] {
|
||||
for backend in [
|
||||
HardwareBackend::Nvenc,
|
||||
HardwareBackend::Qsv,
|
||||
HardwareBackend::Amf,
|
||||
HardwareBackend::Vaapi,
|
||||
HardwareBackend::VideoToolbox,
|
||||
HardwareBackend::Other,
|
||||
] {
|
||||
let mut refresh = new_refresh(VideoSource::Monitor, codec, &sp, &recorder, 0, 1, 1);
|
||||
refresh.hardware_backend = backend;
|
||||
for quality in [
|
||||
0.1,
|
||||
BR_BEST,
|
||||
BR_SPEED,
|
||||
BR_BALANCED,
|
||||
BR_SPEED + 0.01,
|
||||
BR_BEST + 0.1,
|
||||
] {
|
||||
let limit = max_repeat_frames(codec, quality, backend);
|
||||
assert_output_budget(&mut refresh, &sp, quality, limit);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn speed_boundary_uses_the_speed_preset_for_all_backends() {
|
||||
for codec in [
|
||||
CodecFormat::VP8,
|
||||
CodecFormat::VP9,
|
||||
CodecFormat::H264,
|
||||
CodecFormat::H265,
|
||||
] {
|
||||
for backend in [
|
||||
HardwareBackend::Nvenc,
|
||||
HardwareBackend::Qsv,
|
||||
HardwareBackend::Amf,
|
||||
HardwareBackend::Vaapi,
|
||||
HardwareBackend::VideoToolbox,
|
||||
HardwareBackend::Other,
|
||||
] {
|
||||
assert_eq!(
|
||||
max_repeat_frames(codec, BR_SPEED, backend),
|
||||
max_repeat_frames(codec, BR_SPEED + 0.01, backend),
|
||||
"{codec:?} {backend:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(feature = "hwcodec")]
|
||||
fn ram_config_selects_the_actual_backend_and_preserves_other_encoders() {
|
||||
let sp = GenericService::new("repeat-backend-test".to_owned(), false);
|
||||
let recorder = Arc::new(Mutex::new(None));
|
||||
for (name, expected) in [
|
||||
("h264_nvenc", HardwareBackend::Nvenc),
|
||||
("hevc_nvenc", HardwareBackend::Nvenc),
|
||||
("h264_qsv", HardwareBackend::Qsv),
|
||||
("hevc_qsv", HardwareBackend::Qsv),
|
||||
("h264_amf", HardwareBackend::Amf),
|
||||
("hevc_amf", HardwareBackend::Amf),
|
||||
("h264_vaapi", HardwareBackend::Vaapi),
|
||||
("hevc_vaapi", HardwareBackend::Other),
|
||||
("hevc_videotoolbox", HardwareBackend::VideoToolbox),
|
||||
("h264_videotoolbox", HardwareBackend::Other),
|
||||
] {
|
||||
let config = EncoderCfg::HWRAM(scrap::hwcodec::HwRamEncoderConfig {
|
||||
name: name.to_owned(),
|
||||
mc_name: None,
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
quality: BR_BALANCED,
|
||||
keyframe_interval: None,
|
||||
});
|
||||
let codec_format = if name.starts_with("hevc_") {
|
||||
CodecFormat::H265
|
||||
} else {
|
||||
CodecFormat::H264
|
||||
};
|
||||
let mut refresh = StaticRefresh::new(
|
||||
VideoSource::Monitor,
|
||||
codec_format,
|
||||
&config,
|
||||
&sp,
|
||||
&recorder,
|
||||
0,
|
||||
1920,
|
||||
1080,
|
||||
);
|
||||
for quality in [0.1, BR_SPEED, BR_BALANCED, BR_BEST] {
|
||||
let limit = max_repeat_frames(codec_format, quality, expected);
|
||||
assert_output_budget(&mut refresh, &sp, quality, limit);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(feature = "vram")]
|
||||
fn vram_config_resolves_ffmpeg_to_the_actual_hardware_backend() {
|
||||
let sp = GenericService::new("repeat-vram-backend-test".to_owned(), false);
|
||||
let recorder = Arc::new(Mutex::new(None));
|
||||
for (vendor, expected) in [
|
||||
("NV", HardwareBackend::Nvenc),
|
||||
("MFX", HardwareBackend::Qsv),
|
||||
("AMF", HardwareBackend::Amf),
|
||||
("FFMPEG", HardwareBackend::Other),
|
||||
] {
|
||||
let config = EncoderCfg::VRAM(scrap::vram::VRamEncoderConfig {
|
||||
device: Default::default(),
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
quality: BR_BALANCED,
|
||||
keyframe_interval: None,
|
||||
feature: serde_json::from_value(serde_json::json!({
|
||||
"driver": "FFMPEG",
|
||||
"vendor": vendor,
|
||||
"luid": 0,
|
||||
"data_format": "H264",
|
||||
}))
|
||||
.unwrap(),
|
||||
});
|
||||
let mut refresh = StaticRefresh::new(
|
||||
VideoSource::Monitor,
|
||||
CodecFormat::H264,
|
||||
&config,
|
||||
&sp,
|
||||
&recorder,
|
||||
0,
|
||||
1920,
|
||||
1080,
|
||||
);
|
||||
for quality in [0.1, BR_SPEED, BR_BALANCED, BR_BEST] {
|
||||
let limit = max_repeat_frames(CodecFormat::H264, quality, expected);
|
||||
assert_output_budget(&mut refresh, &sp, quality, limit);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod error_tests;
|
||||
@@ -0,0 +1,579 @@
|
||||
use super::{tests::new_refresh, *};
|
||||
use crate::server::{connection::ConnInner, service::Service};
|
||||
use base::message_proto::VideoFrame;
|
||||
use hbb_common::tokio::sync::mpsc;
|
||||
use scrap::{
|
||||
codec::{EncoderApi, EncoderCfg, BR_BALANCED},
|
||||
EncodeYuvFormat,
|
||||
};
|
||||
use std::{cell::RefCell, collections::VecDeque, rc::Rc};
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum Outcome {
|
||||
Frame,
|
||||
Empty,
|
||||
Dropped,
|
||||
Error,
|
||||
}
|
||||
|
||||
struct State {
|
||||
outcomes: VecDeque<Outcome>,
|
||||
calls: usize,
|
||||
disabled: usize,
|
||||
inputs: Vec<Vec<u8>>,
|
||||
}
|
||||
|
||||
struct ScriptedEncoder(Rc<RefCell<State>>);
|
||||
|
||||
impl EncoderApi for ScriptedEncoder {
|
||||
fn new(_: EncoderCfg, _: bool) -> ResultType<Self> {
|
||||
unreachable!()
|
||||
}
|
||||
fn encode_to_message(&mut self, input: EncodeInput, _: i64) -> ResultType<VideoFrame> {
|
||||
let mut state = self.0.borrow_mut();
|
||||
state.calls += 1;
|
||||
if let EncodeInput::YUV(yuv) = input {
|
||||
state.inputs.push(yuv.to_vec());
|
||||
}
|
||||
match state.outcomes.pop_front().expect("unexpected encode") {
|
||||
Outcome::Frame => {
|
||||
let mut vf = VideoFrame::new();
|
||||
vf.set_h264s(Default::default());
|
||||
Ok(vf)
|
||||
}
|
||||
Outcome::Empty => Ok(VideoFrame::new()),
|
||||
Outcome::Dropped => Err(scrap::VpxNoOutput.into()),
|
||||
Outcome::Error => bail!("injected encoding error"),
|
||||
}
|
||||
}
|
||||
fn yuvfmt(&self) -> EncodeYuvFormat {
|
||||
unreachable!()
|
||||
}
|
||||
#[cfg(feature = "vram")]
|
||||
fn input_texture(&self) -> bool {
|
||||
false
|
||||
}
|
||||
fn set_quality(&mut self, _: f32) -> ResultType<()> {
|
||||
unreachable!()
|
||||
}
|
||||
fn bitrate(&self) -> u32 {
|
||||
0
|
||||
}
|
||||
fn support_changing_quality(&self) -> bool {
|
||||
false
|
||||
}
|
||||
fn latency_free(&self) -> bool {
|
||||
true
|
||||
}
|
||||
fn is_hardware(&self) -> bool {
|
||||
true
|
||||
}
|
||||
fn disable(&self) {
|
||||
self.0.borrow_mut().disabled += 1;
|
||||
}
|
||||
}
|
||||
|
||||
fn encoder(outcomes: &[Outcome]) -> (Encoder, Rc<RefCell<State>>) {
|
||||
let state = Rc::new(RefCell::new(State {
|
||||
outcomes: outcomes.iter().copied().collect(),
|
||||
calls: 0,
|
||||
disabled: 0,
|
||||
inputs: Vec::new(),
|
||||
}));
|
||||
(
|
||||
Encoder {
|
||||
codec: Box::new(ScriptedEncoder(Rc::clone(&state))),
|
||||
},
|
||||
state,
|
||||
)
|
||||
}
|
||||
|
||||
fn attempt(refresh: &mut StaticRefresh<'_>, encoder: &mut Encoder) {
|
||||
refresh.elapsed_since_encode = Duration::from_secs(1);
|
||||
refresh
|
||||
.try_encode(
|
||||
&[1],
|
||||
Duration::from_millis(100),
|
||||
BR_BALANCED,
|
||||
Instant::now(),
|
||||
0,
|
||||
encoder,
|
||||
&mut VideoFrameController::new(0),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failures_pause_refresh_until_a_real_frame_succeeds() {
|
||||
use Outcome::*;
|
||||
let sp = GenericService::new("repeat-errors-test".to_owned(), false);
|
||||
let (tx, rx) = mpsc::unbounded_channel();
|
||||
sp.on_subscribe(ConnInner::new(1, None, Some(tx)));
|
||||
let recorder = Arc::new(Mutex::new(None));
|
||||
let mut refresh = new_refresh(
|
||||
VideoSource::Monitor,
|
||||
CodecFormat::H264,
|
||||
&sp,
|
||||
&recorder,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
);
|
||||
let (mut encoder, state) = encoder(&[Error, Error, Empty, Frame, Error, Error, Error, Frame]);
|
||||
refresh.on_frame(&EncodeInput::YUV(&[1]));
|
||||
refresh.on_encoded(true);
|
||||
attempt(&mut refresh, &mut encoder);
|
||||
refresh
|
||||
.try_encode(
|
||||
&[1],
|
||||
Duration::from_millis(100),
|
||||
BR_BALANCED,
|
||||
Instant::now(),
|
||||
0,
|
||||
&mut encoder,
|
||||
&mut VideoFrameController::new(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
state.borrow().calls,
|
||||
1,
|
||||
"failure must retain the rate limit"
|
||||
);
|
||||
attempt(&mut refresh, &mut encoder);
|
||||
attempt(&mut refresh, &mut encoder);
|
||||
assert_eq!(rx.len(), 0);
|
||||
attempt(&mut refresh, &mut encoder);
|
||||
assert_eq!(rx.len(), 1, "empty output must not stop further refreshes");
|
||||
for _ in 0..3 {
|
||||
attempt(&mut refresh, &mut encoder);
|
||||
}
|
||||
attempt(&mut refresh, &mut encoder);
|
||||
assert_eq!(state.borrow().calls, 7);
|
||||
assert_eq!(rx.len(), 1);
|
||||
assert_eq!(state.borrow().disabled, 0);
|
||||
|
||||
refresh.on_frame(&EncodeInput::YUV(&[1]));
|
||||
refresh.on_encoded(false);
|
||||
attempt(&mut refresh, &mut encoder);
|
||||
assert_eq!(
|
||||
state.borrow().calls,
|
||||
7,
|
||||
"a failed real frame must not restart refresh"
|
||||
);
|
||||
refresh.on_frame(&EncodeInput::YUV(&[1]));
|
||||
refresh.on_encoded(true);
|
||||
attempt(&mut refresh, &mut encoder);
|
||||
assert_eq!(state.borrow().calls, 8);
|
||||
assert_eq!(rx.len(), 2);
|
||||
assert_eq!(state.borrow().disabled, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_outputs_exhaust_the_no_output_budget_without_disabling_the_encoder() {
|
||||
let sp = GenericService::new("repeat-empty-test".to_owned(), false);
|
||||
let recorder = Arc::new(Mutex::new(None));
|
||||
let mut refresh = new_refresh(
|
||||
VideoSource::Monitor,
|
||||
CodecFormat::H264,
|
||||
&sp,
|
||||
&recorder,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
);
|
||||
let (mut encoder, state) = encoder(&[Outcome::Empty; 100]);
|
||||
refresh.on_frame(&EncodeInput::YUV(&[1]));
|
||||
refresh.on_encoded(true);
|
||||
for _ in 0..101 {
|
||||
attempt(&mut refresh, &mut encoder);
|
||||
}
|
||||
assert_eq!(state.borrow().calls, 100);
|
||||
assert_eq!(state.borrow().disabled, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn vpx_drops_exhaust_the_no_output_budget_without_disabling_the_encoder() {
|
||||
let sp = GenericService::new("repeat-vpx-drops-test".to_owned(), false);
|
||||
let recorder = Arc::new(Mutex::new(None));
|
||||
for codec in [CodecFormat::VP8, CodecFormat::VP9] {
|
||||
let mut refresh = new_refresh(VideoSource::Monitor, codec, &sp, &recorder, 0, 1, 1);
|
||||
let limit = MAX_REPEAT_NO_OUTPUTS;
|
||||
let (mut encoder, state) = encoder(&vec![Outcome::Dropped; limit]);
|
||||
refresh.on_frame(&EncodeInput::YUV(&[1]));
|
||||
refresh.on_encoded(true);
|
||||
for _ in 0..=limit {
|
||||
attempt(&mut refresh, &mut encoder);
|
||||
}
|
||||
assert_eq!(state.borrow().calls, limit, "{codec:?}");
|
||||
assert_eq!(state.borrow().disabled, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn vpx_dropped_capture_repeats_the_latest_frame() {
|
||||
use super::super::handle_one_frame;
|
||||
use Outcome::*;
|
||||
let sp = GenericService::new("repeat-dropped-capture-test".to_owned(), false);
|
||||
let recorder = Arc::new(Mutex::new(None));
|
||||
for codec in [CodecFormat::VP8, CodecFormat::VP9] {
|
||||
let (tx, mut rx) = mpsc::unbounded_channel();
|
||||
sp.on_subscribe(ConnInner::new(1, None, Some(tx)));
|
||||
let mut refresh = new_refresh(VideoSource::Monitor, codec, &sp, &recorder, 0, 1, 1);
|
||||
let (mut encoder, state) = encoder(&[Frame, Dropped, Dropped, Frame]);
|
||||
refresh.on_frame(&EncodeInput::YUV(&[1]));
|
||||
refresh.on_encoded(true);
|
||||
attempt(&mut refresh, &mut encoder);
|
||||
assert!(rx.try_recv().unwrap().1.has_video_frame());
|
||||
|
||||
let latest_yuv = [2];
|
||||
refresh.on_frame(&EncodeInput::YUV(&latest_yuv));
|
||||
let mut normal_failures = 0;
|
||||
let mut first_frame = false;
|
||||
let mut encoder_warmed_up = false;
|
||||
let result = handle_one_frame(
|
||||
0,
|
||||
&sp,
|
||||
EncodeInput::YUV(&latest_yuv),
|
||||
100,
|
||||
&mut encoder,
|
||||
recorder.clone(),
|
||||
&mut normal_failures,
|
||||
&mut first_frame,
|
||||
&mut encoder_warmed_up,
|
||||
1,
|
||||
1,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(result.send_conn_ids.is_empty());
|
||||
assert!(result.vpx_no_output);
|
||||
refresh.on_frame_encoded(&result);
|
||||
assert_eq!(rx.len(), 0);
|
||||
for (elapsed_ms, expected_calls, expected_outputs) in
|
||||
[(99, 2, 0), (100, 3, 0), (100, 4, 1)]
|
||||
{
|
||||
refresh.elapsed_since_encode = Duration::from_millis(elapsed_ms);
|
||||
refresh
|
||||
.try_encode(
|
||||
&latest_yuv,
|
||||
Duration::from_nanos(33_333_333),
|
||||
BR_BALANCED,
|
||||
Instant::now(),
|
||||
expected_calls as i64 * 100,
|
||||
&mut encoder,
|
||||
&mut VideoFrameController::new(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(state.borrow().calls, expected_calls, "{codec:?}");
|
||||
assert_eq!(rx.len(), expected_outputs);
|
||||
}
|
||||
assert_eq!(state.borrow().inputs, vec![vec![1], vec![2], vec![2], vec![2]]);
|
||||
assert_eq!(state.borrow().disabled, 0);
|
||||
sp.on_unsubscribe(1);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn captured_frame_errors_do_not_enable_repeat() {
|
||||
use super::super::handle_one_frame;
|
||||
let sp = GenericService::new("repeat-capture-error-test".to_owned(), false);
|
||||
let recorder = Arc::new(Mutex::new(None));
|
||||
for (codec, outcome, vpx_no_output) in [
|
||||
(CodecFormat::VP8, Outcome::Error, false),
|
||||
(CodecFormat::VP9, Outcome::Error, false),
|
||||
(CodecFormat::H264, Outcome::Dropped, true),
|
||||
(CodecFormat::H265, Outcome::Dropped, true),
|
||||
] {
|
||||
let mut refresh = new_refresh(VideoSource::Monitor, codec, &sp, &recorder, 0, 1, 1);
|
||||
let (mut encoder, state) = encoder(&[outcome, Outcome::Frame]);
|
||||
refresh.on_frame(&EncodeInput::YUV(&[1]));
|
||||
let mut normal_failures = 0;
|
||||
let mut first_frame = false;
|
||||
let mut encoder_warmed_up = false;
|
||||
let result = handle_one_frame(
|
||||
0,
|
||||
&sp,
|
||||
EncodeInput::YUV(&[1]),
|
||||
0,
|
||||
&mut encoder,
|
||||
recorder.clone(),
|
||||
&mut normal_failures,
|
||||
&mut first_frame,
|
||||
&mut encoder_warmed_up,
|
||||
1,
|
||||
1,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(result.send_conn_ids.is_empty());
|
||||
assert_eq!(result.vpx_no_output, vpx_no_output);
|
||||
refresh.on_frame_encoded(&result);
|
||||
attempt(&mut refresh, &mut encoder);
|
||||
assert_eq!(state.borrow().calls, 1, "{codec:?}");
|
||||
assert_eq!(state.borrow().disabled, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsuccessful_repeats_preserve_the_frame_budget() {
|
||||
let sp = GenericService::new("repeat-frame-budget-test".to_owned(), false);
|
||||
let recorder = Arc::new(Mutex::new(None));
|
||||
for codec in [CodecFormat::VP8, CodecFormat::VP9] {
|
||||
let (tx, rx) = mpsc::unbounded_channel();
|
||||
sp.on_subscribe(ConnInner::new(1, None, Some(tx)));
|
||||
let mut refresh = new_refresh(VideoSource::Monitor, codec, &sp, &recorder, 0, 1, 1);
|
||||
let limit = max_repeat_frames(codec, BR_BALANCED, HardwareBackend::Other);
|
||||
let mut outcomes = vec![Outcome::Dropped; 40];
|
||||
outcomes.extend(vec![Outcome::Empty; 40]);
|
||||
outcomes.extend(vec![Outcome::Frame; limit + 1]);
|
||||
let (mut encoder, state) = encoder(&outcomes);
|
||||
refresh.on_frame(&EncodeInput::YUV(&[1]));
|
||||
refresh.on_encoded(true);
|
||||
for _ in 0..80 {
|
||||
attempt(&mut refresh, &mut encoder);
|
||||
}
|
||||
assert_eq!(rx.len(), 0, "{codec:?}");
|
||||
for _ in 0..=limit {
|
||||
attempt(&mut refresh, &mut encoder);
|
||||
}
|
||||
assert_eq!(state.borrow().calls, 80 + limit, "{codec:?}");
|
||||
assert_eq!(rx.len(), limit);
|
||||
assert_eq!(state.borrow().disabled, 0);
|
||||
sp.on_unsubscribe(1);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeat_intervals_are_codec_specific_after_the_first_attempt() {
|
||||
let sp = GenericService::new("repeat-interval-test".to_owned(), false);
|
||||
let recorder = Arc::new(Mutex::new(None));
|
||||
for (codec, spf_ms, first_interval_ms, repeat_interval_ms) in [
|
||||
(CodecFormat::VP8, 10, 100, 100),
|
||||
(CodecFormat::VP9, 10, 100, 100),
|
||||
(CodecFormat::VP8, 200, 200, 200),
|
||||
(CodecFormat::VP9, 200, 200, 200),
|
||||
(CodecFormat::H264, 10, 100, 34),
|
||||
(CodecFormat::H265, 10, 100, 34),
|
||||
(CodecFormat::H264, 16, 100, 34),
|
||||
(CodecFormat::H265, 16, 100, 34),
|
||||
(CodecFormat::H264, 50, 100, 50),
|
||||
(CodecFormat::H265, 50, 100, 50),
|
||||
(CodecFormat::H264, 200, 200, 200),
|
||||
(CodecFormat::H265, 200, 200, 200),
|
||||
] {
|
||||
for first in [
|
||||
Outcome::Frame,
|
||||
Outcome::Empty,
|
||||
Outcome::Dropped,
|
||||
Outcome::Error,
|
||||
] {
|
||||
let mut refresh = new_refresh(VideoSource::Monitor, codec, &sp, &recorder, 0, 1, 1);
|
||||
let (mut encoder, state) = encoder(&[first, Outcome::Frame]);
|
||||
refresh.on_frame(&EncodeInput::YUV(&[1]));
|
||||
refresh.on_encoded(true);
|
||||
for (elapsed_ms, expected_calls) in [
|
||||
(first_interval_ms - 1, 0),
|
||||
(first_interval_ms, 1),
|
||||
(repeat_interval_ms - 1, 1),
|
||||
(repeat_interval_ms, 2),
|
||||
] {
|
||||
refresh.elapsed_since_encode = Duration::from_millis(elapsed_ms);
|
||||
refresh
|
||||
.try_encode(
|
||||
&[1],
|
||||
Duration::from_millis(spf_ms),
|
||||
BR_BALANCED,
|
||||
Instant::now(),
|
||||
0,
|
||||
&mut encoder,
|
||||
&mut VideoFrameController::new(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(state.borrow().calls, expected_calls, "{codec:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeat_timing_uses_the_capture_loop_start() {
|
||||
let sp = GenericService::new("repeat-loop-clock-test".to_owned(), false);
|
||||
let recorder = Arc::new(Mutex::new(None));
|
||||
let spf = Duration::from_nanos(33_333_333);
|
||||
let clock_start = Instant::now() + Duration::from_secs(1);
|
||||
for first in [
|
||||
Outcome::Frame,
|
||||
Outcome::Empty,
|
||||
Outcome::Dropped,
|
||||
Outcome::Error,
|
||||
] {
|
||||
let mut refresh = new_refresh(
|
||||
VideoSource::Monitor,
|
||||
CodecFormat::H264,
|
||||
&sp,
|
||||
&recorder,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
);
|
||||
let (mut encoder, state) = encoder(&[first, Outcome::Frame]);
|
||||
refresh.on_frame(&EncodeInput::YUV(&[1]));
|
||||
refresh.on_encoded(true);
|
||||
refresh.last_encode = clock_start;
|
||||
for (index, loop_start) in [
|
||||
clock_start + INITIAL_REPEAT_INTERVAL,
|
||||
clock_start + INITIAL_REPEAT_INTERVAL + spf,
|
||||
]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
{
|
||||
// Capture processing finishes partway through the current loop.
|
||||
let capture_end = loop_start + Duration::from_millis(5);
|
||||
refresh.elapsed_since_encode = capture_end.duration_since(refresh.last_encode);
|
||||
refresh
|
||||
.try_encode(
|
||||
&[1],
|
||||
spf,
|
||||
BR_BALANCED,
|
||||
loop_start,
|
||||
0,
|
||||
&mut encoder,
|
||||
&mut VideoFrameController::new(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(state.borrow().calls, index + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_output_budget_is_cumulative_and_restarts_with_a_new_frame() {
|
||||
let sp = GenericService::new("repeat-no-output-budget-test".to_owned(), false);
|
||||
let recorder = Arc::new(Mutex::new(None));
|
||||
for codec in [CodecFormat::VP8, CodecFormat::VP9] {
|
||||
let mut refresh = new_refresh(VideoSource::Monitor, codec, &sp, &recorder, 0, 1, 1);
|
||||
let mut outcomes = Vec::new();
|
||||
for _ in 0..10 {
|
||||
outcomes.push(Outcome::Frame);
|
||||
for _ in 0..5 {
|
||||
outcomes.extend([Outcome::Empty, Outcome::Dropped]);
|
||||
}
|
||||
}
|
||||
outcomes.push(Outcome::Empty);
|
||||
outcomes.push(Outcome::Frame);
|
||||
let (mut encoder, state) = encoder(&outcomes);
|
||||
refresh.on_frame(&EncodeInput::YUV(&[1]));
|
||||
refresh.on_encoded(true);
|
||||
for _ in 0..111 {
|
||||
attempt(&mut refresh, &mut encoder);
|
||||
}
|
||||
assert_eq!(state.borrow().calls, 110, "{codec:?}");
|
||||
assert_eq!(state.borrow().disabled, 0);
|
||||
|
||||
refresh.on_frame(&EncodeInput::YUV(&[1]));
|
||||
refresh.on_encoded(false);
|
||||
attempt(&mut refresh, &mut encoder);
|
||||
assert_eq!(state.borrow().calls, 110);
|
||||
refresh.on_encoded(true);
|
||||
attempt(&mut refresh, &mut encoder);
|
||||
assert_eq!(state.borrow().calls, 111);
|
||||
attempt(&mut refresh, &mut encoder);
|
||||
assert_eq!(state.borrow().calls, 112);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeat_error_limits_are_codec_specific() {
|
||||
let sp = GenericService::new("repeat-codec-errors-test".to_owned(), false);
|
||||
let recorder = Arc::new(Mutex::new(None));
|
||||
for (codec, expected_calls) in [
|
||||
(CodecFormat::VP8, 100),
|
||||
(CodecFormat::VP9, 100),
|
||||
(CodecFormat::AV1, 0),
|
||||
(CodecFormat::H264, 3),
|
||||
(CodecFormat::H265, 3),
|
||||
] {
|
||||
let mut refresh = new_refresh(VideoSource::Monitor, codec, &sp, &recorder, 0, 1, 1);
|
||||
let limit = MAX_REPEAT_NO_OUTPUTS;
|
||||
let (mut encoder, state) = encoder(&vec![Outcome::Error; limit]);
|
||||
refresh.on_frame(&EncodeInput::YUV(&[1]));
|
||||
refresh.on_encoded(true);
|
||||
for _ in 0..=limit {
|
||||
attempt(&mut refresh, &mut encoder);
|
||||
}
|
||||
assert_eq!(state.borrow().calls, expected_calls, "{codec:?}");
|
||||
assert_eq!(state.borrow().disabled, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeats_leave_normal_failure_tracking_and_switching_intact() {
|
||||
use super::super::handle_one_frame;
|
||||
use Outcome::*;
|
||||
let sp = GenericService::new("repeat-normal-errors-test".to_owned(), false);
|
||||
let recorder = Arc::new(Mutex::new(None));
|
||||
let mut refresh = new_refresh(
|
||||
VideoSource::Monitor,
|
||||
CodecFormat::H264,
|
||||
&sp,
|
||||
&recorder,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
);
|
||||
let normal_limit = if cfg!(target_os = "android") { 9 } else { 3 };
|
||||
let mut outcomes = vec![Error, Error, Frame];
|
||||
outcomes.extend(vec![Error; normal_limit - 1]);
|
||||
let (mut encoder, state) = encoder(&outcomes);
|
||||
refresh.on_frame(&EncodeInput::YUV(&[1]));
|
||||
refresh.on_encoded(true);
|
||||
let mut normal_failures = 0;
|
||||
let mut first_frame = false;
|
||||
let mut encoder_warmed_up = false;
|
||||
handle_one_frame(
|
||||
0,
|
||||
&sp,
|
||||
EncodeInput::YUV(&[1]),
|
||||
0,
|
||||
&mut encoder,
|
||||
recorder.clone(),
|
||||
&mut normal_failures,
|
||||
&mut first_frame,
|
||||
&mut encoder_warmed_up,
|
||||
1,
|
||||
1,
|
||||
)
|
||||
.unwrap();
|
||||
attempt(&mut refresh, &mut encoder);
|
||||
attempt(&mut refresh, &mut encoder);
|
||||
assert_eq!(state.borrow().disabled, 0);
|
||||
for _ in 2..normal_limit {
|
||||
handle_one_frame(
|
||||
0,
|
||||
&sp,
|
||||
EncodeInput::YUV(&[1]),
|
||||
0,
|
||||
&mut encoder,
|
||||
recorder.clone(),
|
||||
&mut normal_failures,
|
||||
&mut first_frame,
|
||||
&mut encoder_warmed_up,
|
||||
1,
|
||||
1,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let result = handle_one_frame(
|
||||
0,
|
||||
&sp,
|
||||
EncodeInput::YUV(&[1]),
|
||||
0,
|
||||
&mut encoder,
|
||||
recorder.clone(),
|
||||
&mut normal_failures,
|
||||
&mut first_frame,
|
||||
&mut encoder_warmed_up,
|
||||
1,
|
||||
1,
|
||||
);
|
||||
assert_eq!(result.unwrap_err().to_string(), "SWITCH");
|
||||
assert_eq!(state.borrow().disabled, 1);
|
||||
}
|
||||
Reference in New Issue
Block a user