fix(ffmpeg): handle QSV encoder stalls and undersized buffers (#16303)

* fix(ffmpeg): bound QSV encoder retries

Limit QSV frame submission to 100 retries and synchronization to 10
retries so persistent SDK warnings return an error to the caller. Retain
queued buffers when synchronization reaches its retry limit, and allow
later calls to retry without latching an error.

Wait 1 ms between retryable submission warnings on Windows: the original
av_usleep(500) becomes Sleep(0), exhausting the retry budget before even
a brief device-busy interval can end. Preserve non-Windows wait behavior
and the existing negative SDK result handling.

Validated against FFmpeg 7.1.1 on Intel UHD 730 with 42 fault/control
cases, synchronization-timeout recovery and close cleanup, real H.264
and HEVC encoding, and separate QSV hardware decoding. Four concurrent
1080p encoders completed 2400 frames without errors.

* fix(ffmpeg): handle undersized QSV HEVC buffers

Reserve at least three bytes per aligned pixel for HEVC output, following
Intel's documented workaround for underestimated bitstream capacity. Check
the size before assigning it to AVPacket's integer capacity.

Report negative synchronization results before reading output metadata or
draining the FIFO, keeping failed tasks owned until encoder close.

On Intel UHD 730, the existing code fails at input 242 with real desktop
captures; the patched code encodes and decodes all 1,200 inputs. Inputs and
PTS match, and the first 241 encoded frames are byte-identical. H.264 also
passes 300 frames, and restoring the small buffer verifies error handling.

* fix(ffmpeg): refresh QSV HEVC capacity after reset and shorten sync waits

* fix(ffmpeg): apply QSV buffer sizing to H.264

* fix: backport FFmpeg hybrid MP4 finalization fix

Backport FFmpeg commit 36896af64a9ecf6835758186e7161a2e954c364d so hybrid MP4 finalization writes the final moov before hiding the original fragmented index. Preserve recoverable fragments when the final index cannot be written.

res/vcpkg/ffmpeg/patch/0013-fix-qsv-encode-timeouts.patch

@@ -0,0 +1,67 @@
+diff --git a/libavcodec/qsvenc.c b/libavcodec/qsvenc.c
+--- a/libavcodec/qsvenc.c
++++ b/libavcodec/qsvenc.c
+@@ -43,6 +43,9 @@
+ #include "qsv_internal.h"
+ #include "qsvenc.h"
+ #include "refstruct.h"
++
++#define QSV_MAX_ENCODE_RETRIES 100
++#define QSV_SYNC_TIMEOUT_US 2000000
+ 
+ struct profile_names {
+     mfxU16 profile;
+@@ -2468,6 +2471,7 @@
+     QSVFrame *qsv_frame = NULL;
+     mfxEncodeCtrl* enc_ctrl = NULL;
+     int ret;
++    int retries = 0;
+ 
+     if (frame) {
+         ret = submit_frame(q, frame, &qsv_frame);
+@@ -2538,6 +2542,18 @@
+ 
+     do {
+         ret = MFXVideoENCODE_EncodeFrameAsync(q->session, enc_ctrl, surf, pkt.bs, pkt.sync);
++        if ((ret == MFX_WRN_DEVICE_BUSY || ret == MFX_WRN_IN_EXECUTION) &&
++            retries++ >= QSV_MAX_ENCODE_RETRIES) {
++            av_log(avctx, AV_LOG_ERROR, "QSV encoder submission retry limit exceeded\n");
++            ret = AVERROR_EXTERNAL;
++            goto free;
++        }
++#ifdef _WIN32
++        // Sub-millisecond waits become Sleep(0) on Windows.
++        if (ret == MFX_WRN_DEVICE_BUSY || ret == MFX_WRN_IN_EXECUTION)
++            av_usleep(1000);
++#else
+         if (ret == MFX_WRN_DEVICE_BUSY)
+             av_usleep(500);
++#endif
+     } while (ret == MFX_WRN_DEVICE_BUSY || ret == MFX_WRN_IN_EXECUTION);
+@@ -2656,12 +2672,23 @@
+         mfxExtAVCEncodedFrameInfo *enc_info;
+         mfxExtBuffer **enc_buf;
+         enum AVPictureType pict_type;
+-
+-        av_fifo_read(q->async_fifo, &qpkt, 1);
++        int64_t sync_deadline = av_gettime_relative() + QSV_SYNC_TIMEOUT_US;
++
++        // The SDK may still use these buffers after a synchronization timeout.
++        av_fifo_peek(q->async_fifo, &qpkt, 1, 0);
+ 
+         do {
+-            ret = MFXVideoCORE_SyncOperation(q->session, *qpkt.sync, 1000);
++            int64_t remaining = sync_deadline - av_gettime_relative();
++            if (remaining <= 0) {
++                av_log(avctx, AV_LOG_ERROR, "QSV encoder synchronization timed out\n");
++                return AVERROR_EXTERNAL;
++            }
++            ret = MFXVideoCORE_SyncOperation(q->session, *qpkt.sync,
++                                            FFMIN(1000, (remaining + 999) / 1000));
++            if (ret == MFX_WRN_IN_EXECUTION)
++                av_usleep(1000);
+         } while (ret == MFX_WRN_IN_EXECUTION);
++        av_fifo_drain2(q->async_fifo, 1);
+ 
+         qpkt.pkt.dts  = av_rescale_q(qpkt.bs->DecodeTimeStamp, (AVRational){1, 90000}, avctx->time_base);
+         qpkt.pkt.pts  = av_rescale_q(qpkt.bs->TimeStamp,       (AVRational){1, 90000}, avctx->time_base);

res/vcpkg/ffmpeg/patch/0014-fix-qsv-encode-buffer.patch

@@ -0,0 +1,64 @@
+diff --git a/libavcodec/qsvenc.c b/libavcodec/qsvenc.c
+--- a/libavcodec/qsvenc.c
++++ b/libavcodec/qsvenc.c
+@@ -1419,6 +1419,18 @@
+     return 0;
+ }
+ 
++static int qsv_update_packet_size(QSVEncContext *q, const mfxInfoMFX *mfx)
++{
++    // Intel documents a larger buffer for complex CQP frames.
++    // https://github.com/Intel-Media-SDK/MediaSDK/blob/master/doc/mediasdk_release_notes.md
++    int64_t packet_size = (int64_t)mfx->BufferSizeInKB * mfx->BRCParamMultiplier * 1000 * 4;
++
++    if (packet_size > INT_MAX - AV_INPUT_BUFFER_PADDING_SIZE)
++        return AVERROR(ENOMEM);
++    q->packet_size = FFMAX(q->packet_size, packet_size);
++    return 0;
++}
++
+ static int qsv_retrieve_enc_params(AVCodecContext *avctx, QSVEncContext *q)
+ {
+     AVCPBProperties *cpb_props;
+@@ -1508,7 +1520,13 @@
+         return ff_qsv_print_error(avctx, ret,
+                                   "Error calling GetVideoParam");
+ 
+-    q->packet_size = q->param.mfx.BufferSizeInKB * q->param.mfx.BRCParamMultiplier * 1000;
++    if (avctx->codec_id == AV_CODEC_ID_H264 || avctx->codec_id == AV_CODEC_ID_HEVC) {
++        ret = qsv_update_packet_size(q, &q->param.mfx);
++        if (ret < 0)
++            return ret;
++    } else {
++        q->packet_size = q->param.mfx.BufferSizeInKB * q->param.mfx.BRCParamMultiplier * 1000;
++    }
+ 
+     if (!extradata.SPSBufSize || (need_pps && !extradata.PPSBufSize)
+         || (q->hevc_vps && !extradata_vps.VPSBufSize)
+@@ -2650,6 +2668,17 @@
+     if (ret < 0)
+         return ff_qsv_print_error(avctx, ret, "Error during resetting");
+ 
++    if (avctx->codec_id == AV_CODEC_ID_H264 || avctx->codec_id == AV_CODEC_ID_HEVC) {
++        mfxVideoParam param = { 0 };
++
++        ret = MFXVideoENCODE_GetVideoParam(q->session, &param);
++        if (ret < 0)
++            return ff_qsv_print_error(avctx, ret, "Error querying parameters after reset");
++        ret = qsv_update_packet_size(q, &param.mfx);
++        if (ret < 0)
++            return ret;
++    }
++
+     return 0;
+ }
+ 
+@@ -2688,6 +2717,8 @@
+             if (ret == MFX_WRN_IN_EXECUTION)
+                 av_usleep(1000);
+         } while (ret == MFX_WRN_IN_EXECUTION);
++        if (ret < 0)
++            return ff_qsv_print_error(avctx, ret, "Error synchronizing encoded frame");
+         av_fifo_drain2(q->async_fifo, 1);
+ 
+         qpkt.pkt.dts  = av_rescale_q(qpkt.bs->DecodeTimeStamp, (AVRational){1, 90000}, avctx->time_base);

res/vcpkg/ffmpeg/patch/0015-fix-hybrid-mp4-finalization.patch

@@ -0,0 +1,89 @@
+From: Martin Storsjö
+Subject: movenc: Make the hybrid_fragmented mode more robust
+Source: https://github.com/FFmpeg/FFmpeg/commit/36896af64a9ecf6835758186e7161a2e954c364d
+
+movenc: Make the hybrid_fragmented mode more robust
+
+Write the moov tag at the end first, before overwriting the mdat size
+at the start of the file.
+
+In case writing the final moov box fails (e.g. due to being out
+of disk), we haven't broken the initial moov box yet.
+
+Thus if writing stops between these steps, we could end up with
+a file with two moov boxes - which arguably is more feasible to
+recover from, than from a file with no moov boxes at all.
+
+diff --git a/libavformat/movenc.c b/libavformat/movenc.c
+--- a/libavformat/movenc.c
++++ b/libavformat/movenc.c
+@@ -8546,6 +8546,28 @@ static int shift_data(AVFormatContext *s)
+     return ff_format_shift_data(s, mov->reserved_header_pos, moov_size);
+ }
+ 
++static void mov_write_mdat_size(AVFormatContext *s)
++{
++    MOVMuxContext *mov = s->priv_data;
++    AVIOContext *pb = s->pb;
++
++    /* Write size of mdat tag */
++    if (mov->mdat_size + 8 <= UINT32_MAX) {
++        avio_seek(pb, mov->mdat_pos, SEEK_SET);
++        avio_wb32(pb, mov->mdat_size + 8);
++        if (mov->flags & FF_MOV_FLAG_HYBRID_FRAGMENTED)
++            ffio_wfourcc(pb, "mdat"); // overwrite the original moov into a mdat
++    } else {
++        /* overwrite 'wide' placeholder atom */
++        avio_seek(pb, mov->mdat_pos - 8, SEEK_SET);
++        /* special value: real atom size will be 64 bit value after
++         * tag field */
++        avio_wb32(pb, 1);
++        ffio_wfourcc(pb, "mdat");
++        avio_wb64(pb, mov->mdat_size + 16);
++    }
++}
++
+ static int mov_write_trailer(AVFormatContext *s)
+ {
+     MOVMuxContext *mov = s->priv_data;
+@@ -8605,21 +8627,9 @@ static int mov_write_trailer(AVFormatContext *s)
+ 
+         moov_pos = avio_tell(pb);
+ 
+-        /* Write size of mdat tag */
+-        if (mov->mdat_size + 8 <= UINT32_MAX) {
+-            avio_seek(pb, mov->mdat_pos, SEEK_SET);
+-            avio_wb32(pb, mov->mdat_size + 8);
+-            if (mov->flags & FF_MOV_FLAG_HYBRID_FRAGMENTED)
+-                ffio_wfourcc(pb, "mdat"); // overwrite the original moov into a mdat
+-        } else {
+-            /* overwrite 'wide' placeholder atom */
+-            avio_seek(pb, mov->mdat_pos - 8, SEEK_SET);
+-            /* special value: real atom size will be 64 bit value after
+-             * tag field */
+-            avio_wb32(pb, 1);
+-            ffio_wfourcc(pb, "mdat");
+-            avio_wb64(pb, mov->mdat_size + 16);
+-        }
++        if (!(mov->flags & FF_MOV_FLAG_HYBRID_FRAGMENTED))
++            mov_write_mdat_size(s);
++
+         avio_seek(pb, mov->reserved_moov_size > 0 ? mov->reserved_header_pos : moov_pos, SEEK_SET);
+ 
+         if (mov->flags & FF_MOV_FLAG_FASTSTART) {
+@@ -8647,6 +8657,15 @@ static int mov_write_trailer(AVFormatContext *s)
+             if ((res = mov_write_moov_tag(pb, mov, s)) < 0)
+                 return res;
+         }
++
++        if (mov->flags & FF_MOV_FLAG_HYBRID_FRAGMENTED) {
++            // With hybrid fragmentation, only write the mdat size (hiding
++            // the original moov and all the fragments within the mdat)
++            // after we've successfully written the complete moov, to avoid
++            // risk for an unreadable file if writing the final moov fails.
++            mov_write_mdat_size(s);
++        }
++
+         res = 0;
+     } else {
+         mov_auto_flush_fragment(s, 1);

res/vcpkg/ffmpeg/portfile.cmake

@@ -30,6 +30,9 @@ vcpkg_from_github(
     patch/0010.disable-loading-DLLs-from-app-dir.patch
     patch/0011-android-mediacodec-encode-align-64.patch
     patch/0012-fix-macos-big-sur-CVBufferCopyAttachments.patch
+    patch/0013-fix-qsv-encode-timeouts.patch
+    patch/0014-fix-qsv-encode-buffer.patch
+    patch/0015-fix-hybrid-mp4-finalization.patch
 )
 
 if(SOURCE_PATH MATCHES " ")