PR #23531 opened by Jun Zhao (mypopydev) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23531 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23531.patch
The capture callback dropped the previous frame instead of waiting for it to be consumed, so frames were lost when avf_read_packet() fell behind; 39fbd06314 (return EAGAIN instead of waiting) made it easy to hit. Add back a condition variable: the capture callback blocks until avf_read_packet() takes the current frame, and the reader waits on it when no frame is ready. An is_stopping flag set in destroy_context() wakes both sides so teardown cannot deadlock. observed_quit is now set under the lock and broadcast as well, so a blocked reader wakes and returns EOF when a transport-control device stops delivering frames. Based on a patch by Zhongxin Zhuang <[email protected]>; here unlock_frames() only broadcasts and unlocks, so a read no longer releases the other stream's still-unconsumed frame. Verified by capturing camera+mic for 8s and comparing delivered packet counts before/after this change: ffmpeg -f avfoundation -pixel_format nv12 -framerate 30 -i "0:0" -t 8 \ -vf "scale=2560:1440,hqdn3d" -c:v libx264 -preset medium \ -c:a aac out.mp4 ffprobe -count_packets -show_entries stream=nb_read_packets out.mp4 slow consumer (ideal: video ~240, audio ~375) video audio before (EAGAIN) 174 163 (audio ~57% dropped) after (condvar) 234 376 (no drops) fast consumer (-preset ultrafast, no filter) before 240 328 after 238 376 Signed-off-by: Jun Zhao <[email protected]> # Summary of changes Briefly describe what this PR does and why. <!-- If this PR requires new FATE test samples, attach them to the PR and list their target paths below (relative to the fate-suite root). Attached filenames must match the sample's filename: ```fate-samples # e.g. vorbis/new-sample.ogg ``` --> >From 388eaee4fde6cc5ee8ee297588a3cd907fe0bfcf Mon Sep 17 00:00:00 2001 From: Jun Zhao <[email protected]> Date: Thu, 18 Jun 2026 23:14:21 +0800 Subject: [PATCH] avdevice/avfoundation: wait for frame consumption to avoid dropping A/V frames The capture callback dropped the previous frame instead of waiting for it to be consumed, so frames were lost when avf_read_packet() fell behind; 39fbd06314 (return EAGAIN instead of waiting) made it easy to hit. Add back a condition variable: the capture callback blocks until avf_read_packet() takes the current frame, and the reader waits on it when no frame is ready. An is_stopping flag set in destroy_context() wakes both sides so teardown cannot deadlock. observed_quit is now set under the lock and broadcast as well, so a blocked reader wakes and returns EOF when a transport-control device stops delivering frames. Based on a patch by Zhongxin Zhuang <[email protected]>; here unlock_frames() only broadcasts and unlocks, so a read no longer releases the other stream's still-unconsumed frame. Verified by capturing camera+mic for 8s and comparing delivered packet counts before/after this change: ffmpeg -f avfoundation -pixel_format nv12 -framerate 30 -i "0:0" -t 8 \ -vf "scale=2560:1440,hqdn3d" -c:v libx264 -preset medium \ -c:a aac out.mp4 ffprobe -count_packets -show_entries stream=nb_read_packets out.mp4 slow consumer (ideal: video ~240, audio ~375) video audio before (EAGAIN) 174 163 (audio ~57% dropped) after (condvar) 234 376 (no drops) fast consumer (-preset ultrafast, no filter) before 240 328 after 238 376 Signed-off-by: Jun Zhao <[email protected]> --- libavdevice/avfoundation.m | 56 ++++++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m index ebec1ac4f2..6f52989fc9 100644 --- a/libavdevice/avfoundation.m +++ b/libavdevice/avfoundation.m @@ -89,6 +89,8 @@ typedef struct int frames_captured; int audio_frames_captured; pthread_mutex_t frame_lock; + pthread_cond_t frame_wait_cond; + int is_stopping; id avf_delegate; id avf_audio_delegate; @@ -147,6 +149,7 @@ static void lock_frames(AVFContext* ctx) static void unlock_frames(AVFContext* ctx) { + pthread_cond_broadcast(&ctx->frame_wait_cond); pthread_mutex_unlock(&ctx->frame_lock); } @@ -210,7 +213,12 @@ static void unlock_frames(AVFContext* ctx) if (mode != _context->observed_mode) { if (mode == AVCaptureDeviceTransportControlsNotPlayingMode) { + // Set under the lock and broadcast so a reader blocked in + // avf_read_packet() wakes up and returns EOF instead of + // hanging once the device stops delivering frames. + lock_frames(_context); _context->observed_quit = 1; + unlock_frames(_context); } _context->observed_mode = mode; } @@ -229,8 +237,13 @@ static void unlock_frames(AVFContext* ctx) { lock_frames(_context); - if (_context->current_frame != nil) { - CFRelease(_context->current_frame); + while ((_context->current_frame != nil) && !_context->is_stopping) { + pthread_cond_wait(&_context->frame_wait_cond, &_context->frame_lock); + } + + if (_context->is_stopping) { + unlock_frames(_context); + return; } _context->current_frame = (CMSampleBufferRef)CFRetain(videoFrame); @@ -273,8 +286,13 @@ static void unlock_frames(AVFContext* ctx) { lock_frames(_context); - if (_context->current_audio_frame != nil) { - CFRelease(_context->current_audio_frame); + while ((_context->current_audio_frame != nil) && !_context->is_stopping) { + pthread_cond_wait(&_context->frame_wait_cond, &_context->frame_lock); + } + + if (_context->is_stopping) { + unlock_frames(_context); + return; } _context->current_audio_frame = (CMSampleBufferRef)CFRetain(audioFrame); @@ -288,6 +306,12 @@ static void unlock_frames(AVFContext* ctx) static void destroy_context(AVFContext* ctx) { + // Wake any capture callback blocked waiting for the consumer and make it + // bail out, so stopRunning() can drain the session without a deadlock. + lock_frames(ctx); + ctx->is_stopping = 1; + unlock_frames(ctx); + [ctx->capture_session stopRunning]; [ctx->capture_session release]; @@ -305,10 +329,17 @@ static void destroy_context(AVFContext* ctx) av_freep(&ctx->url); av_freep(&ctx->audio_buffer); + pthread_cond_destroy(&ctx->frame_wait_cond); pthread_mutex_destroy(&ctx->frame_lock); if (ctx->current_frame) { CFRelease(ctx->current_frame); + ctx->current_frame = nil; + } + + if (ctx->current_audio_frame) { + CFRelease(ctx->current_audio_frame); + ctx->current_audio_frame = nil; } } @@ -836,6 +867,8 @@ static int avf_read_header(AVFormatContext *s) ctx->num_video_devices = [devices count] + [devices_muxed count]; pthread_mutex_init(&ctx->frame_lock, NULL); + pthread_cond_init(&ctx->frame_wait_cond, NULL); + ctx->is_stopping = 0; #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070 CGGetActiveDisplayList(0, NULL, &num_screens); @@ -1120,10 +1153,10 @@ static int avf_read_packet(AVFormatContext *s, AVPacket *pkt) { AVFContext* ctx = (AVFContext*)s->priv_data; + lock_frames(ctx); do { CVImageBufferRef image_buffer; CMBlockBufferRef block_buffer; - lock_frames(ctx); if (ctx->current_frame != nil) { int status; @@ -1253,16 +1286,21 @@ static int avf_read_packet(AVFormatContext *s, AVPacket *pkt) ctx->current_audio_frame = nil; } else { pkt->data = NULL; - unlock_frames(ctx); if (ctx->observed_quit) { + unlock_frames(ctx); return AVERROR_EOF; - } else { - return AVERROR(EAGAIN); } + // No frame available yet: wait until a capture callback delivers + // one (or until the device is being torn down). + pthread_cond_wait(&ctx->frame_wait_cond, &ctx->frame_lock); } + } while (!pkt->data && !ctx->is_stopping); + if (ctx->is_stopping) { unlock_frames(ctx); - } while (!pkt->data); + return AVERROR_EOF; + } + unlock_frames(ctx); return 0; } -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
