PR #23471 opened by commodo URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23471 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23471.patch
The main idea here is to help V4L2 M2M devices (especially VPU encoders) to obtain buffers (from other V4L2 devices) without any copying in-between. On iMX8 devices this saves about 17 ms per frames when moving a frame from a V4L2 camera device to the encoder (as V4L2 M2M device). Then the VPU encoder takes about 3-5 ms to encode (depending on tuning parameters of the encoder). For `libavdevice/v4l2.c` a `-dmabuf` option is added (open for suggestions to change the name), because there is no simple way yet, to determine whether a consumer can process DRM_PRIME exported buffers. A working example looks like this `ffmpeg -f v4l2 -dmabuf 1 -video_size 1920x1080 -pixel_format nv12 -i /dev/video2 -c:v h264_v4l2m2m -f mpegts "udp://192.168.50.222:1234"` >From b23ce4d01a0ff33bdd1cbcb48ad9c988d05212f8 Mon Sep 17 00:00:00 2001 From: Alexandru Ardelean <[email protected]> Date: Wed, 10 Jun 2026 15:23:45 +0300 Subject: [PATCH 1/4] avcodec/v4l2_m2m_enc: handle DRM_PRIME buffers Accept AV_PIX_FMT_DRM_PRIME frames on the OUTPUT queue and import their DMABuf fds (V4L2_MEMORY_DMABUF), avoiding a copy of the raw pixel data. Signed-off-by: Alexandru Ardelean <[email protected]> --- libavcodec/v4l2_buffers.c | 43 ++++++++++++++++++++++++++++++++++++++- libavcodec/v4l2_context.c | 4 ++-- libavcodec/v4l2_context.h | 7 +++++++ libavcodec/v4l2_m2m_enc.c | 27 ++++++++++++++++++++++-- 4 files changed, 76 insertions(+), 5 deletions(-) diff --git a/libavcodec/v4l2_buffers.c b/libavcodec/v4l2_buffers.c index f96c26771b..1f1898d6d3 100644 --- a/libavcodec/v4l2_buffers.c +++ b/libavcodec/v4l2_buffers.c @@ -29,6 +29,7 @@ #include <poll.h> #include "libavcodec/avcodec.h" #include "libavutil/attributes.h" +#include "libavutil/hwcontext_drm.h" #include "libavutil/pixdesc.h" #include "libavutil/refstruct.h" #include "v4l2_context.h" @@ -433,8 +434,37 @@ static int v4l2_buffer_swframe_to_buf(const AVFrame *frame, V4L2Buffer *out) int ff_v4l2_buffer_avframe_to_buf(const AVFrame *frame, V4L2Buffer *out) { + V4L2Context *ctx = out->context; + int i; + v4l2_set_pts(out, frame->pts); + if (frame->format == AV_PIX_FMT_DRM_PRIME && frame->data[0]) { + const AVDRMFrameDescriptor *desc = (const AVDRMFrameDescriptor *)frame->data[0]; + + if (V4L2_TYPE_IS_MULTIPLANAR(out->buf.type)) { + for (i = 0; i < desc->nb_objects; i++) { + out->planes[i].m.fd = desc->objects[i].fd; + out->planes[i].length = (uint32_t)desc->objects[i].size; + out->planes[i].bytesused = (uint32_t)desc->objects[i].size; + } + out->buf.m.planes = out->planes; + out->buf.length = desc->nb_objects; + } else { + out->buf.m.fd = desc->objects[0].fd; + out->buf.length = (uint32_t)desc->objects[0].size; + out->buf.bytesused = (uint32_t)desc->objects[0].size; + } + return 0; + } + + if (ctx->use_dmabuf) { + av_log(logger(out), AV_LOG_ERROR, + "%s: use_dmabuf set but frame is not AV_PIX_FMT_DRM_PRIME (format=%d)\n", + ctx->name, frame->format); + return AVERROR(EINVAL); + } + return v4l2_buffer_swframe_to_buf(frame, out); } @@ -520,7 +550,7 @@ int ff_v4l2_buffer_initialize(V4L2Buffer* avbuf, int index) V4L2Context *ctx = avbuf->context; int ret, i; - avbuf->buf.memory = V4L2_MEMORY_MMAP; + avbuf->buf.memory = ctx->use_dmabuf ? V4L2_MEMORY_DMABUF : V4L2_MEMORY_MMAP; avbuf->buf.type = ctx->type; avbuf->buf.index = index; @@ -540,6 +570,8 @@ int ff_v4l2_buffer_initialize(V4L2Buffer* avbuf, int index) if (avbuf->buf.m.planes[i].length) avbuf->num_planes++; } + if (!avbuf->num_planes) + return AVERROR(EINVAL); } else avbuf->num_planes = 1; @@ -549,6 +581,15 @@ int ff_v4l2_buffer_initialize(V4L2Buffer* avbuf, int index) ctx->format.fmt.pix_mp.plane_fmt[i].bytesperline : ctx->format.fmt.pix.bytesperline; + if (ctx->use_dmabuf) { + /* No mmap for DMABuf import; record expected size from format */ + avbuf->plane_info[i].length = V4L2_TYPE_IS_MULTIPLANAR(ctx->type) ? + ctx->format.fmt.pix_mp.plane_fmt[i].sizeimage : + ctx->format.fmt.pix.sizeimage; + avbuf->plane_info[i].mm_addr = NULL; + continue; + } + if (V4L2_TYPE_IS_MULTIPLANAR(ctx->type)) { avbuf->plane_info[i].length = avbuf->buf.m.planes[i].length; avbuf->plane_info[i].mm_addr = mmap(NULL, avbuf->buf.m.planes[i].length, diff --git a/libavcodec/v4l2_context.c b/libavcodec/v4l2_context.c index be1df3785b..c2145f45cc 100644 --- a/libavcodec/v4l2_context.c +++ b/libavcodec/v4l2_context.c @@ -444,7 +444,7 @@ static V4L2Buffer* v4l2_getfree_v4l2buf(V4L2Context *ctx) static int v4l2_release_buffers(V4L2Context* ctx) { struct v4l2_requestbuffers req = { - .memory = V4L2_MEMORY_MMAP, + .memory = ctx->use_dmabuf ? V4L2_MEMORY_DMABUF : V4L2_MEMORY_MMAP, .type = ctx->type, .count = 0, /* 0 -> unmaps buffers from the driver */ }; @@ -727,7 +727,7 @@ int ff_v4l2_context_init(V4L2Context* ctx) memset(&req, 0, sizeof(req)); req.count = ctx->num_buffers; - req.memory = V4L2_MEMORY_MMAP; + req.memory = ctx->use_dmabuf ? V4L2_MEMORY_DMABUF : V4L2_MEMORY_MMAP; req.type = ctx->type; ret = ioctl(s->fd, VIDIOC_REQBUFS, &req); if (ret < 0) { diff --git a/libavcodec/v4l2_context.h b/libavcodec/v4l2_context.h index fdd5cf5284..532295eaca 100644 --- a/libavcodec/v4l2_context.h +++ b/libavcodec/v4l2_context.h @@ -93,6 +93,13 @@ typedef struct V4L2Context { */ int done; + /** + * If set, the OUTPUT context will import DMABuf fds rather than copying + * pixel data. Must be set before ff_v4l2_context_init() is called. + * Only meaningful for OUTPUT (encoder input) contexts. + */ + int use_dmabuf; + } V4L2Context; /** diff --git a/libavcodec/v4l2_m2m_enc.c b/libavcodec/v4l2_m2m_enc.c index 93703ccc63..9f2ae67c7c 100644 --- a/libavcodec/v4l2_m2m_enc.c +++ b/libavcodec/v4l2_m2m_enc.c @@ -26,10 +26,12 @@ #include <search.h> #include "encode.h" #include "libavcodec/avcodec.h" +#include "libavutil/hwcontext_drm.h" #include "libavutil/pixdesc.h" #include "libavutil/pixfmt.h" #include "libavutil/opt.h" #include "codec_internal.h" +#include "hwconfig.h" #include "profiles.h" #include "v4l2_context.h" #include "v4l2_m2m.h" @@ -336,6 +338,7 @@ static av_cold int v4l2_encode_init(AVCodecContext *avctx) V4L2Context *capture, *output; V4L2m2mContext *s; V4L2m2mPriv *priv = avctx->priv_data; + enum AVPixelFormat sw_pix_fmt; enum AVPixelFormat pix_fmt_output; uint32_t v4l2_fmt_output; int ret; @@ -347,13 +350,27 @@ static av_cold int v4l2_encode_init(AVCodecContext *avctx) capture = &s->capture; output = &s->output; + if (avctx->pix_fmt == AV_PIX_FMT_DRM_PRIME) { + AVHWFramesContext *frames_ctx; + if (!avctx->hw_frames_ctx) { + av_log(avctx, AV_LOG_ERROR, + "hw_frames_ctx must be set when using DRM_PRIME frames as input\n"); + return AVERROR(EINVAL); + } + frames_ctx = (AVHWFramesContext *)avctx->hw_frames_ctx->data; + sw_pix_fmt = frames_ctx->sw_format; + output->use_dmabuf = 1; + } else { + sw_pix_fmt = avctx->pix_fmt; + } + /* common settings output/capture */ output->height = capture->height = avctx->height; output->width = capture->width = avctx->width; /* output context */ output->av_codec_id = AV_CODEC_ID_RAWVIDEO; - output->av_pix_fmt = avctx->pix_fmt; + output->av_pix_fmt = sw_pix_fmt; /* capture context */ capture->av_codec_id = avctx->codec_id; @@ -372,7 +389,7 @@ static av_cold int v4l2_encode_init(AVCodecContext *avctx) v4l2_fmt_output = output->format.fmt.pix.pixelformat; pix_fmt_output = ff_v4l2_format_v4l2_to_avfmt(v4l2_fmt_output, AV_CODEC_ID_RAWVIDEO); - if (pix_fmt_output != avctx->pix_fmt) { + if (pix_fmt_output != sw_pix_fmt) { const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt_output); av_log(avctx, AV_LOG_ERROR, "Encoder requires %s pixel format.\n", desc->name); return AVERROR(EINVAL); @@ -411,6 +428,11 @@ static const FFCodecDefault v4l2_m2m_defaults[] = { { NULL }, }; +static const AVCodecHWConfigInternal *const v4l2m2m_enc_hw_configs[] = { + HW_CONFIG_ENCODER_FRAMES(DRM_PRIME, DRM), + NULL, +}; + #define M2MENC_CLASS(NAME, OPTIONS_NAME) \ static const AVClass v4l2_m2m_ ## NAME ## _enc_class = { \ .class_name = #NAME "_v4l2m2m_encoder", \ @@ -437,6 +459,7 @@ static const FFCodecDefault v4l2_m2m_defaults[] = { .caps_internal = FF_CODEC_CAP_NOT_INIT_THREADSAFE | \ FF_CODEC_CAP_INIT_CLEANUP, \ .p.wrapper_name = "v4l2m2m", \ + .hw_configs = v4l2m2m_enc_hw_configs, \ } M2MENC(mpeg4,"MPEG4", mpeg4_options, AV_CODEC_ID_MPEG4); -- 2.52.0 >From 21dfc88acb9d1bec4991f82499ff954252bf054a Mon Sep 17 00:00:00 2001 From: Alexandru Ardelean <[email protected]> Date: Wed, 10 Jun 2026 15:23:45 +0300 Subject: [PATCH 2/4] avcodec/v4l2_m2m_enc: support AV_CODEC_CAP_ENCODER_REORDERED_OPAQUE Advertise the capability and propagate per-frame opaque values to the output packets under AV_CODEC_FLAG_COPY_OPAQUE. Only the buffer timestamp round-trips through the queues, so the values are stored keyed by the source frame pts and re-attached on dequeue. Signed-off-by: Alexandru Ardelean <[email protected]> --- libavcodec/v4l2_m2m.h | 11 ++++++ libavcodec/v4l2_m2m_enc.c | 73 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 81 insertions(+), 3 deletions(-) diff --git a/libavcodec/v4l2_m2m.h b/libavcodec/v4l2_m2m.h index 4ba33dc335..46a3442b7f 100644 --- a/libavcodec/v4l2_m2m.h +++ b/libavcodec/v4l2_m2m.h @@ -40,6 +40,13 @@ { "num_output_buffers", "Number of buffers in the output context",\ OFFSET(num_output_buffers), AV_OPT_TYPE_INT, { .i64 = 16 }, 2, INT_MAX, FLAGS } +/* AV_CODEC_FLAG_COPY_OPAQUE: input opaque kept keyed by pts. */ +typedef struct V4L2m2mOpaque { + int64_t pts; + void *opaque; + AVBufferRef *opaque_ref; +} V4L2m2mOpaque; + typedef struct V4L2m2mContext { char devname[PATH_MAX]; int fd; @@ -48,6 +55,10 @@ typedef struct V4L2m2mContext { V4L2Context capture; V4L2Context output; + V4L2m2mOpaque *opaque_map; + int nb_opaque_map; + int opaque_map_size; + /* dynamic stream reconfig */ AVCodecContext *avctx; sem_t refsync; diff --git a/libavcodec/v4l2_m2m_enc.c b/libavcodec/v4l2_m2m_enc.c index 9f2ae67c7c..62dca6b57d 100644 --- a/libavcodec/v4l2_m2m_enc.c +++ b/libavcodec/v4l2_m2m_enc.c @@ -27,6 +27,7 @@ #include "encode.h" #include "libavcodec/avcodec.h" #include "libavutil/hwcontext_drm.h" +#include "libavutil/mem.h" #include "libavutil/pixdesc.h" #include "libavutil/pixfmt.h" #include "libavutil/opt.h" @@ -273,6 +274,56 @@ static int v4l2_prepare_encoder(V4L2m2mContext *s) return 0; } +/* Stash an input frame's opaque values keyed by pts; best effort. */ +static void v4l2_enc_opaque_store(V4L2m2mContext *s, const AVFrame *frame) +{ + AVRational tb; + int64_t pts; + V4L2m2mOpaque *e; + + if (s->nb_opaque_map == s->opaque_map_size) { + int n = s->opaque_map_size ? s->opaque_map_size * 2 : 16; + V4L2m2mOpaque *m = av_realloc_array(s->opaque_map, n, sizeof(*m)); + if (!m) + return; + s->opaque_map = m; + s->opaque_map_size = n; + } + + /* key on the us-round-tripped pts the packet will carry, else non-us + * timebases never match on fetch and opaque_refs leak until close */ + tb = s->avctx->pkt_timebase.num ? s->avctx->pkt_timebase : s->avctx->time_base; + pts = frame->pts == AV_NOPTS_VALUE ? 0 : frame->pts; + if (tb.num > 0 && tb.den > 0) { + AVRational us = av_make_q(1, 1000000); + pts = av_rescale_q(av_rescale_q(pts, tb, us), us, tb); + } + + e = &s->opaque_map[s->nb_opaque_map]; + e->pts = pts; + e->opaque = frame->opaque; + e->opaque_ref = NULL; + if (frame->opaque_ref) { + e->opaque_ref = av_buffer_ref(frame->opaque_ref); + if (!e->opaque_ref) + return; + } + s->nb_opaque_map++; +} + +/* Re-attach the stored opaque values matching pkt->pts to the packet. */ +static void v4l2_enc_opaque_fetch(V4L2m2mContext *s, AVPacket *pkt) +{ + for (int i = 0; i < s->nb_opaque_map; i++) { + if (s->opaque_map[i].pts == pkt->pts) { + pkt->opaque = s->opaque_map[i].opaque; + pkt->opaque_ref = s->opaque_map[i].opaque_ref; + s->opaque_map[i] = s->opaque_map[--s->nb_opaque_map]; + return; + } + } +} + static int v4l2_send_frame(AVCodecContext *avctx, const AVFrame *frame) { V4L2m2mContext *s = ((V4L2m2mPriv*)avctx->priv_data)->context; @@ -307,8 +358,11 @@ static int v4l2_receive_packet(AVCodecContext *avctx, AVPacket *avpkt) } ret = v4l2_send_frame(avctx, frame); - if (ret != AVERROR(EAGAIN)) + if (ret != AVERROR(EAGAIN)) { + if (frame && ret >= 0 && (avctx->flags & AV_CODEC_FLAG_COPY_OPAQUE)) + v4l2_enc_opaque_store(s, frame); av_frame_unref(frame); + } if (ret < 0 && ret != AVERROR(EAGAIN)) return ret; @@ -330,7 +384,10 @@ static int v4l2_receive_packet(AVCodecContext *avctx, AVPacket *avpkt) } dequeue: - return ff_v4l2_context_dequeue_packet(capture, avpkt); + ret = ff_v4l2_context_dequeue_packet(capture, avpkt); + if (ret >= 0 && (avctx->flags & AV_CODEC_FLAG_COPY_OPAQUE)) + v4l2_enc_opaque_fetch(s, avpkt); + return ret; } static av_cold int v4l2_encode_init(AVCodecContext *avctx) @@ -400,6 +457,15 @@ static av_cold int v4l2_encode_init(AVCodecContext *avctx) static av_cold int v4l2_encode_close(AVCodecContext *avctx) { + V4L2m2mContext *s = ((V4L2m2mPriv*)avctx->priv_data)->context; + + if (s) { + for (int i = 0; i < s->nb_opaque_map; i++) + av_buffer_unref(&s->opaque_map[i].opaque_ref); + av_freep(&s->opaque_map); + s->nb_opaque_map = s->opaque_map_size = 0; + } + return ff_v4l2_m2m_codec_end(avctx->priv_data); } @@ -454,7 +520,8 @@ static const AVCodecHWConfigInternal *const v4l2m2m_enc_hw_configs[] = { FF_CODEC_RECEIVE_PACKET_CB(v4l2_receive_packet), \ .close = v4l2_encode_close, \ .defaults = v4l2_m2m_defaults, \ - .p.capabilities = AV_CODEC_CAP_HARDWARE | AV_CODEC_CAP_DELAY, \ + .p.capabilities = AV_CODEC_CAP_HARDWARE | AV_CODEC_CAP_DELAY | \ + AV_CODEC_CAP_ENCODER_REORDERED_OPAQUE, \ .color_ranges = AVCOL_RANGE_MPEG, \ .caps_internal = FF_CODEC_CAP_NOT_INIT_THREADSAFE | \ FF_CODEC_CAP_INIT_CLEANUP, \ -- 2.52.0 >From ab1e88876521b82c79a3a0aec3037e176d89d3f5 Mon Sep 17 00:00:00 2001 From: Alexandru Ardelean <[email protected]> Date: Wed, 10 Jun 2026 15:23:45 +0300 Subject: [PATCH 3/4] avutil/hwcontext: allow hwcontext_drm to be compiled in without libdrm libdrm is only used for drmGetVersion(); drop the build dependency so hwcontext_drm is always available, e.g. on bare aarch64 cross-builds. Signed-off-by: Alexandru Ardelean <[email protected]> --- libavutil/Makefile | 2 +- libavutil/hwcontext.c | 2 -- libavutil/hwcontext_drm.c | 9 +++++++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/libavutil/Makefile b/libavutil/Makefile index 2e8a5de551..692cf022b0 100644 --- a/libavutil/Makefile +++ b/libavutil/Makefile @@ -210,7 +210,7 @@ OBJS-$(CONFIG_D3D11VA) += hwcontext_d3d11va.o OBJS-$(CONFIG_D3D12VA) += hwcontext_d3d12va.o OBJS-$(CONFIG_DXVA2) += hwcontext_dxva2.o OBJS-$(CONFIG_AMF) += hwcontext_amf.o -OBJS-$(CONFIG_LIBDRM) += hwcontext_drm.o +OBJS += hwcontext_drm.o OBJS-$(CONFIG_MACOS_KPERF) += macos_kperf.o OBJS-$(CONFIG_MEDIACODEC) += hwcontext_mediacodec.o OBJS-$(CONFIG_OHCODEC) += hwcontext_oh.o diff --git a/libavutil/hwcontext.c b/libavutil/hwcontext.c index 83bd7457e8..f810f8d703 100644 --- a/libavutil/hwcontext.c +++ b/libavutil/hwcontext.c @@ -39,9 +39,7 @@ static const HWContextType * const hw_table[] = { #if CONFIG_D3D12VA &ff_hwcontext_type_d3d12va, #endif -#if CONFIG_LIBDRM &ff_hwcontext_type_drm, -#endif #if CONFIG_DXVA2 &ff_hwcontext_type_dxva2, #endif diff --git a/libavutil/hwcontext_drm.c b/libavutil/hwcontext_drm.c index 565c02dead..42b111a535 100644 --- a/libavutil/hwcontext_drm.c +++ b/libavutil/hwcontext_drm.c @@ -30,8 +30,10 @@ #include <sys/ioctl.h> #endif +#if CONFIG_LIBDRM #include <drm.h> #include <xf86drm.h> +#endif #include "avassert.h" #include "hwcontext.h" @@ -52,12 +54,15 @@ static int drm_device_create(AVHWDeviceContext *hwdev, const char *device, AVDictionary *opts, int flags) { AVDRMDeviceContext *hwctx = hwdev->hwctx; +#if CONFIG_LIBDRM drmVersionPtr version; +#endif hwctx->fd = open(device, O_RDWR); if (hwctx->fd < 0) return AVERROR(errno); +#if CONFIG_LIBDRM version = drmGetVersion(hwctx->fd); if (!version) { av_log(hwdev, AV_LOG_ERROR, "Failed to get version information " @@ -72,6 +77,10 @@ static int drm_device_create(AVHWDeviceContext *hwdev, const char *device, version->version_patchlevel); drmFreeVersion(version); +#else + av_log(hwdev, AV_LOG_INFO, "Opened DRM device %s " + "(libdrm not available; skipping version probe).\n", device); +#endif hwdev->free = &drm_device_free; -- 2.52.0 >From 05678b1fc251d816c9d6b327bec1531c210b27d6 Mon Sep 17 00:00:00 2001 From: Alexandru Ardelean <[email protected]> Date: Wed, 10 Jun 2026 18:43:00 +0300 Subject: [PATCH 4/4] avdevice/v4l2: export captured frames as DRM_PRIME for zero-copy With the new "dmabuf" option each captured buffer is exported via VIDIOC_EXPBUF and emitted as a wrapped-avframe DRM_PRIME frame, letting a V4L2 encoder import it without copying the pixel data. Signed-off-by: Alexandru Ardelean <[email protected]> --- libavdevice/v4l2.c | 247 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 246 insertions(+), 1 deletion(-) diff --git a/libavdevice/v4l2.c b/libavdevice/v4l2.c index 2ca093b7b0..cb295b0b3a 100644 --- a/libavdevice/v4l2.c +++ b/libavdevice/v4l2.c @@ -34,6 +34,8 @@ #include "libavutil/avassert.h" #include "libavutil/avstring.h" +#include "libavutil/hwcontext.h" +#include "libavutil/hwcontext_drm.h" #include "libavutil/imgutils.h" #include "libavutil/mem.h" #include "libavutil/parseutils.h" @@ -110,6 +112,11 @@ struct video_data { int list_format; /**< Set by a private option. */ int list_standard; /**< Set by a private option. */ char *framerate; /**< Set by a private option. */ + int use_dmabuf; /**< Set by a private option. */ + enum AVPixelFormat sw_pix_fmt; + uint32_t drm_format; + AVBufferRef *hwdevice; + AVBufferRef *hwframes; int use_libv4l2; int (*open_f)(const char *file, int oflag, ...); @@ -541,6 +548,155 @@ static int convert_timestamp(AVFormatContext *ctx, int64_t *ts) return 0; } +/* V4L2 and DRM name RGB formats by opposite byte order, hence the RGB/BGR swap. */ +static uint32_t v4l2_pixfmt_to_drm(uint32_t v4l2_fmt) +{ + static const struct { + uint32_t v4l2_fmt; + uint32_t drm_fmt; + } map[] = { + { V4L2_PIX_FMT_NV12, MKTAG('N', 'V', '1', '2') }, + { V4L2_PIX_FMT_NV21, MKTAG('N', 'V', '2', '1') }, + { V4L2_PIX_FMT_YUV420, MKTAG('Y', 'U', '1', '2') }, + { V4L2_PIX_FMT_YVU420, MKTAG('Y', 'V', '1', '2') }, + { V4L2_PIX_FMT_NV16, MKTAG('N', 'V', '1', '6') }, + { V4L2_PIX_FMT_NV61, MKTAG('N', 'V', '6', '1') }, + { V4L2_PIX_FMT_YUYV, MKTAG('Y', 'U', 'Y', 'V') }, + { V4L2_PIX_FMT_YVYU, MKTAG('Y', 'V', 'Y', 'U') }, + { V4L2_PIX_FMT_UYVY, MKTAG('U', 'Y', 'V', 'Y') }, + { V4L2_PIX_FMT_VYUY, MKTAG('V', 'Y', 'U', 'Y') }, + { V4L2_PIX_FMT_NV24, MKTAG('N', 'V', '2', '4') }, + { V4L2_PIX_FMT_NV42, MKTAG('N', 'V', '4', '2') }, + { V4L2_PIX_FMT_RGB565, MKTAG('R', 'G', '1', '6') }, + { V4L2_PIX_FMT_BGR24, MKTAG('R', 'G', '2', '4') }, + { V4L2_PIX_FMT_RGB24, MKTAG('B', 'G', '2', '4') }, + }; + int i; + + for (i = 0; i < FF_ARRAY_ELEMS(map); i++) + if (map[i].v4l2_fmt == v4l2_fmt) + return map[i].drm_fmt; + + return 0; +} + +/* desc must be first: a (AVDRMFrameDescriptor *) cast of this struct is used. */ +typedef struct V4L2DRMFrame { + AVDRMFrameDescriptor desc; + struct video_data *s; + int index; +} V4L2DRMFrame; + +static void v4l2_free_drm_desc(void *opaque, uint8_t *data) +{ + V4L2DRMFrame *f = (V4L2DRMFrame *)data; + struct video_data *s = f->s; + struct v4l2_plane planes[VIDEO_MAX_PLANES]; + struct v4l2_buffer buf = { + .type = s->buf_type, + .memory = V4L2_MEMORY_MMAP, + .index = f->index, + .m.planes = s->multiplanar ? planes : NULL, + .length = s->multiplanar ? VIDEO_MAX_PLANES : 0, + }; + int i; + + enqueue_buffer(s, &buf); + for (i = 0; i < f->desc.nb_objects; i++) + close(f->desc.objects[i].fd); + av_free(f); +} + +static void v4l2_free_drm_frame(void *opaque, uint8_t *data) +{ + AVFrame *frame = (AVFrame *)data; + av_frame_free(&frame); +} + +static int v4l2_buffer_export_drm(AVFormatContext *ctx, const struct v4l2_buffer *vbuf, + AVFrame **out_frame) +{ + struct video_data *s = ctx->priv_data; + int max_planes = s->multiplanar ? VIDEO_MAX_PLANES : 1; + AVDRMFrameDescriptor *desc; + V4L2DRMFrame *f; + AVFrame *frame; + int i, n, ret; + + f = av_mallocz(sizeof(*f)); + if (!f) + return AVERROR(ENOMEM); + f->s = s; + f->index = vbuf->index; + desc = &f->desc; + + for (n = 0; n < max_planes && n < AV_DRM_MAX_PLANES; n++) { + struct v4l2_exportbuffer expbuf = { + .type = s->buf_type, + .index = vbuf->index, + .plane = n, + }; + if (v4l2_ioctl(s->fd, VIDIOC_EXPBUF, &expbuf) < 0) { + if (n == 0) { + ret = AVERROR(errno); + goto fail; + } + break; + } + desc->objects[n].fd = expbuf.fd; + desc->objects[n].size = s->buf_data[vbuf->index].len; + desc->objects[n].format_modifier = 0; + } + desc->nb_objects = n; + + desc->nb_layers = 1; + desc->layers[0].format = s->drm_format; + desc->layers[0].nb_planes = n; + for (i = 0; i < n; i++) { + desc->layers[0].planes[i].object_index = i; + desc->layers[0].planes[i].offset = 0; + desc->layers[0].planes[i].pitch = s->width; + } + + frame = av_frame_alloc(); + if (!frame) { + ret = AVERROR(ENOMEM); + goto fail_fds; + } + + frame->format = AV_PIX_FMT_DRM_PRIME; + frame->width = s->width; + frame->height = s->height; + frame->hw_frames_ctx = av_buffer_ref(s->hwframes); + if (!frame->hw_frames_ctx) { + av_frame_free(&frame); + ret = AVERROR(ENOMEM); + goto fail_fds; + } + + /* buf[0] (owns f) must be created last: its destructor re-enqueues the + * buffer, so no earlier error path may run it (would double-enqueue). */ + frame->buf[0] = av_buffer_create((uint8_t *)f, sizeof(*f), + v4l2_free_drm_desc, NULL, 0); + if (!frame->buf[0]) { + av_frame_free(&frame); + ret = AVERROR(ENOMEM); + goto fail_fds; + } + + frame->data[0] = (uint8_t *)desc; + + *out_frame = frame; + return 0; + +fail_fds: + for (i = 0; i < n; i++) + close(desc->objects[i].fd); +fail: + av_free(f); + return ret; +} + static int mmap_read_frame(AVFormatContext *ctx, AVPacket *pkt) { struct video_data *s = ctx->priv_data; @@ -603,8 +759,27 @@ static int mmap_read_frame(AVFormatContext *ctx, AVPacket *pkt) } } + if (s->use_dmabuf) { + AVFrame *frame; + + res = v4l2_buffer_export_drm(ctx, &buf, &frame); + if (res < 0) { + enqueue_buffer(s, &buf); + return res; + } + + pkt->buf = av_buffer_create((uint8_t *)frame, sizeof(*frame), + v4l2_free_drm_frame, NULL, 0); + if (!pkt->buf) { + av_frame_free(&frame); + return AVERROR(ENOMEM); + } + pkt->data = (uint8_t *)frame; + pkt->size = sizeof(*frame); + pkt->flags |= AV_PKT_FLAG_TRUSTED; + } /* Image is at s->buf_data[buf.index].start */ - if (atomic_load(&s->buffers_queued) == FFMAX(s->buffers / 8, 1)) { + else if (atomic_load(&s->buffers_queued) == FFMAX(s->buffers / 8, 1)) { /* when we start getting low on queued buffers, fall back on copying data */ res = av_new_packet(pkt, bytesused); if (res < 0) { @@ -879,6 +1054,55 @@ static int v4l2_read_probe(const AVProbeData *p) return 0; } +static int v4l2_dmabuf_init(AVFormatContext *ctx) +{ + struct video_data *s = ctx->priv_data; + AVHWFramesContext *frames; + AVHWDeviceContext *device; + AVDRMDeviceContext *drm; + int ret; + + s->drm_format = v4l2_pixfmt_to_drm(s->pixelformat); + if (!s->drm_format) { + av_log(ctx, AV_LOG_ERROR, + "No DRM fourcc known for V4L2 format %s; cannot export DRM_PRIME\n", + av_fourcc2str(s->pixelformat)); + return AVERROR(ENOSYS); + } + + s->hwdevice = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_DRM); + if (!s->hwdevice) + return AVERROR(ENOMEM); + device = (AVHWDeviceContext *)s->hwdevice->data; + drm = device->hwctx; + drm->fd = -1; + ret = av_hwdevice_ctx_init(s->hwdevice); + if (ret < 0) + goto fail; + + s->hwframes = av_hwframe_ctx_alloc(s->hwdevice); + if (!s->hwframes) { + ret = AVERROR(ENOMEM); + goto fail; + } + frames = (AVHWFramesContext *)s->hwframes->data; + frames->format = AV_PIX_FMT_DRM_PRIME; + frames->sw_format = s->sw_pix_fmt; + frames->width = s->width; + frames->height = s->height; + + ret = av_hwframe_ctx_init(s->hwframes); + if (ret < 0) + goto fail; + + return 0; + +fail: + av_buffer_unref(&s->hwframes); + av_buffer_unref(&s->hwdevice); + return ret; +} + static int v4l2_read_header(AVFormatContext *ctx) { struct video_data *s = ctx->priv_data; @@ -1028,6 +1252,23 @@ static int v4l2_read_header(AVFormatContext *ctx) if (st->avg_frame_rate.den) st->codecpar->bit_rate = s->frame_size * av_q2d(st->avg_frame_rate) * 8; + if (s->use_dmabuf) { + if (codec_id != AV_CODEC_ID_RAWVIDEO || + st->codecpar->format == AV_PIX_FMT_NONE) { + av_log(ctx, AV_LOG_ERROR, + "dmabuf export requires a raw (uncompressed) pixel format\n"); + res = AVERROR(EINVAL); + goto fail; + } + s->sw_pix_fmt = st->codecpar->format; + if ((res = v4l2_dmabuf_init(ctx)) < 0) + goto fail; + + st->codecpar->codec_id = AV_CODEC_ID_WRAPPED_AVFRAME; + st->codecpar->format = AV_PIX_FMT_DRM_PRIME; + st->codecpar->codec_tag = 0; + } + return 0; fail: @@ -1056,6 +1297,9 @@ static int v4l2_read_close(AVFormatContext *ctx) mmap_close(s); + av_buffer_unref(&s->hwframes); + av_buffer_unref(&s->hwdevice); + ff_timefilter_destroy(s->timefilter); v4l2_close(s->fd); return 0; @@ -1167,6 +1411,7 @@ static const AVOption options[] = { { "abs", "use absolute timestamps (wall clock)", OFFSET(ts_mode), AV_OPT_TYPE_CONST, {.i64 = V4L_TS_ABS }, 0, 2, DEC, .unit = "timestamps" }, { "mono2abs", "force conversion from monotonic to absolute timestamps", OFFSET(ts_mode), AV_OPT_TYPE_CONST, {.i64 = V4L_TS_MONO2ABS }, 0, 2, DEC, .unit = "timestamps" }, { "use_libv4l2", "use libv4l2 (v4l-utils) conversion functions", OFFSET(use_libv4l2), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, DEC }, + { "dmabuf", "export capture buffers as DRM_PRIME frames", OFFSET(use_dmabuf), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, DEC }, { NULL }, }; -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
