PR #24355 opened by Daniel Stadelmann (dstadelmann-iis) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24355 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24355.patch
This bitstream filter allows for interactivity with the Fraunhofer IIS mpeghdec UI Manager by reporting the current AudioScene XML and accepting ActionEvent XML to pass to the wrapped UI Manager. Signed-off-by: Stadelmann, Daniel <[email protected]> <!-- If this PR requires new FATE test samples, attach them to the PR and list their target paths below (relative to the fate-suite root). Attached filenames must match the sample's filename: ```fate-samples # e.g. vorbis/new-sample.ogg ``` --> >From d86e3bb44365d3621d862cba97304675df75080e Mon Sep 17 00:00:00 2001 From: "Stadelmann, Daniel" <[email protected]> Date: Tue, 21 Jul 2026 10:06:18 +0200 Subject: [PATCH] libavcodec/bsf/libmpeghdec_uimanager: Add MPEG-H 3DA UI Manager bitstream filter This bitstream filter allows for interactivity with the Fraunhofer IIS mpeghdec UI Manager by reporting the current AudioScene XML and accepting ActionEvent XML to pass to the wrapped UI Manager. Signed-off-by: Stadelmann, Daniel <[email protected]> --- configure | 4 +- doc/bitstream_filters.texi | 38 +++ libavcodec/bitstream_filters.c | 1 + libavcodec/bsf/Makefile | 1 + libavcodec/bsf/libmpeghdec_uimanager.c | 307 +++++++++++++++++++++++++ 5 files changed, 350 insertions(+), 1 deletion(-) create mode 100644 libavcodec/bsf/libmpeghdec_uimanager.c diff --git a/configure b/configure index 5fecdfd898..19e68f36db 100755 --- a/configure +++ b/configure @@ -3789,6 +3789,7 @@ h264_metadata_bsf_select="cbs_h264" h264_redundant_pps_bsf_select="cbs_h264" hevc_metadata_bsf_select="cbs_h265" lcevc_metadata_bsf_select="cbs_lcevc" +libmpeghdec_uimanager_bsf_deps="libmpeghdec" mjpeg2jpeg_bsf_select="jpegtables" mpeg2_metadata_bsf_select="cbs_mpeg2" smpte436m_to_eia608_bsf_select="smpte_436m" @@ -3873,6 +3874,7 @@ libmodplug_demuxer_deps="libmodplug" libmp3lame_encoder_deps="libmp3lame" libmp3lame_encoder_select="audio_frame_queue mpegaudioheader" libmpeghdec_decoder_deps="libmpeghdec" +libmpeghdec_decoder_select="libmpeghdec_uimanager_bsf" liboapv_encoder_deps="liboapv" libopencore_amrnb_decoder_deps="libopencore_amrnb" libopencore_amrnb_encoder_deps="libopencore_amrnb" @@ -7403,7 +7405,7 @@ fi enabled libmodplug && require_pkg_config libmodplug libmodplug libmodplug/modplug.h ModPlug_Load enabled libmp3lame && require "libmp3lame >= 3.98.3" lame/lame.h lame_set_VBR_quality -lmp3lame $libm_extralibs -enabled libmpeghdec && require_pkg_config libmpeghdec "mpeghdec >= 3.0.0" mpeghdec/mpeghdecoder.h mpeghdecoder_init +enabled libmpeghdec && require_pkg_config libmpeghdec "mpeghdec >= 4.0.0" mpeghdec/mpeghdecoder.h mpeghdecoder_init enabled libmysofa && { check_pkg_config libmysofa libmysofa mysofa.h mysofa_neighborhood_init_withstepdefine || require libmysofa mysofa.h mysofa_neighborhood_init_withstepdefine -lmysofa $zlib_extralibs; } enabled libonnxruntime && require libonnxruntime onnxruntime_c_api.h OrtGetApiBase -lonnxruntime diff --git a/doc/bitstream_filters.texi b/doc/bitstream_filters.texi index c6705bbd39..8cff7c692e 100644 --- a/doc/bitstream_filters.texi +++ b/doc/bitstream_filters.texi @@ -612,6 +612,44 @@ For example, to remux 30 MB/sec NTSC IMX to MOV: ffmpeg -i input.mxf -c copy -bsf:v imxdump -tag:v mx3n output.mov @end example +@section libmpeghdec_uimanager + +libmpeghdec UI Manager wrapper. + +libmpeghdec_uimanager allows libmpeghdec to process MPEG-H 3D Audio UI +Manager packets. +Requires the presence of the libmpeghdec headers and library during +configuration. You need to explicitly configure the build with +@code{--enable-libmpeghdec --enable-nonfree}. + +@table @option +@item interactivity +Enable or disable interactivity, allowing or disallowing inserting of ActionEvent +XML messages. Is disabled automatically if MPEG-H 3DA UI Manager packets are +already present in the input bitstream. Default is enabled. + +@item audio_scene +Read-only option to receive the latest parsed (currently active) AudioScene XML. +This options is overridden by the libmpeghdec_uimanager bitstream filter on new +AudioScenes. + +@item audio_scene_file +File path to write all AudioScenes into. For every new/changed AudioScene, its +XML representation is appended to this file. Can be used together with +@option{audio_scene}. + +@item action_event +An ActionEvent XML string to be applied on the next iteration of the bitstream +filter. This option is cleared by the libmpeghdec_uimanager bitstream filter +after the ActionEvent has been applied and can then be re-set by the caller. + +@item persistency_file +File path for the UI Manager persistency file. On initialization of the +bitstream filter, this file is read (if it exists) and its contents are set +as UI Manager persistency storage. On destruction, the bitstream filter writes +the persistency storage back into this file. +@end table + @section mjpeg2jpeg Convert MJPEG/AVI1 packets to full JPEG/JFIF packets. diff --git a/libavcodec/bitstream_filters.c b/libavcodec/bitstream_filters.c index a9f3218f0f..c2d6174463 100644 --- a/libavcodec/bitstream_filters.c +++ b/libavcodec/bitstream_filters.c @@ -51,6 +51,7 @@ extern const FFBitStreamFilter ff_hevc_mp4toannexb_bsf; extern const FFBitStreamFilter ff_imx_dump_header_bsf; extern const FFBitStreamFilter ff_lcevc_merge_bsf; extern const FFBitStreamFilter ff_lcevc_metadata_bsf; +extern const FFBitStreamFilter ff_libmpeghdec_uimanager_bsf; extern const FFBitStreamFilter ff_media100_to_mjpegb_bsf; extern const FFBitStreamFilter ff_mjpeg2jpeg_bsf; extern const FFBitStreamFilter ff_mjpega_dump_header_bsf; diff --git a/libavcodec/bsf/Makefile b/libavcodec/bsf/Makefile index abbd4be49e..0f99772cff 100644 --- a/libavcodec/bsf/Makefile +++ b/libavcodec/bsf/Makefile @@ -32,6 +32,7 @@ OBJS-$(CONFIG_HEVC_MP4TOANNEXB_BSF) += bsf/hevc_mp4toannexb.o OBJS-$(CONFIG_IMX_DUMP_HEADER_BSF) += bsf/imx_dump_header.o OBJS-$(CONFIG_LCEVC_MERGE_BSF) += bsf/lcevc_merge.o OBJS-$(CONFIG_LCEVC_METADATA_BSF) += bsf/lcevc_metadata.o +OBJS-$(CONFIG_LIBMPEGHDEC_UIMANAGER_BSF) += bsf/libmpeghdec_uimanager.o OBJS-$(CONFIG_MEDIA100_TO_MJPEGB_BSF) += bsf/media100_to_mjpegb.o OBJS-$(CONFIG_MJPEG2JPEG_BSF) += bsf/mjpeg2jpeg.o OBJS-$(CONFIG_MJPEGA_DUMP_HEADER_BSF) += bsf/mjpega_dump_header.o diff --git a/libavcodec/bsf/libmpeghdec_uimanager.c b/libavcodec/bsf/libmpeghdec_uimanager.c new file mode 100644 index 0000000000..a2a1537973 --- /dev/null +++ b/libavcodec/bsf/libmpeghdec_uimanager.c @@ -0,0 +1,307 @@ +/* + * MPEG-H 3D Audio Decoder Wrapper + * Copyright (C) 2026 Fraunhofer Institute for Integrated Circuits IIS + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/* + * Please note that this FFmpeg Software is licensed under the LGPL-2.1 + * but is combined with software that is licensed under different terms, namely + * the "Software License for The Fraunhofer FDK MPEG-H Software". Fraunhofer + * as the initial licensor does not interpret the LGPL-2.1 as requiring + * distribution of the MPEG-H Software under the LGPL-2.1 if being distributed + * together with this FFmpeg Software. Therefore, downstream distribution of + * FFmpeg Software does not imply any right to redistribute the MPEG-H Software + * under the LGPL-2.1. + */ +#include <stdbool.h> +#include <string.h> +#include <mpeghdec/mpeghUIManager.h> + +#include "libavcodec/bsf.h" +#include "libavcodec/bsf_internal.h" + +#include "libavutil/file_open.h" +#include "libavutil/mem.h" +#include "libavutil/opt.h" + +#define MAX_PACKET_SIZE (64 * 1024) +#define XML_CHUNK_SIZE 4096 +#define PERSISTENCY_SIZE USHRT_MAX + +typedef struct MPEGHUIManagerContext { + const AVClass *class; + + HANDLE_MPEGH_UI_MANAGER handle; + + int interactivity; + char *audio_scene_xml; + char *action_event_xml; + char *audio_scene_file_path; + FILE *audio_scene_file; + + char *persistency_file_path; + void *persistency_buffer; +} MPEGHUIManagerContext; + +static void libmpeghdec_uimanager_close(AVBSFContext *bsfc); + +static int read_persistency_file(AVBSFContext *bsfc) +{ + MPEGHUIManagerContext *ctx = bsfc->priv_data; + FILE *persistency_file = NULL; + int err = 0; + size_t persistency_size = PERSISTENCY_SIZE; + + if (!ctx->persistency_file_path) { + return 0; + } + + ctx->persistency_buffer = av_mallocz(persistency_size); + + if (strcmp(ctx->persistency_file_path, "-")) { + persistency_file = avpriv_fopen_utf8(ctx->persistency_file_path, "rb"); + if (!persistency_file && errno == ENOENT) { + av_log(bsfc, AV_LOG_INFO, "Persistency file does not yet exist, using empty buffer: %s\n", ctx->persistency_file_path); + } else if (!persistency_file) { + err = AVERROR(errno); + av_log(bsfc, AV_LOG_ERROR, "Could not open persistency file %s: %s\n", ctx->persistency_file_path, av_err2str(err)); + libmpeghdec_uimanager_close(bsfc); + return err; + } else { + persistency_size = fread(ctx->persistency_buffer, 1, persistency_size, persistency_file); + err = ferror(persistency_file); + if (strcmp(ctx->persistency_file_path, "-")) { + fclose(persistency_file); + } + if (err) { + err = AVERROR(errno); + av_log(bsfc, AV_LOG_ERROR, "Could not read persistency file %s: %s\n", ctx->persistency_file_path, av_err2str(err)); + libmpeghdec_uimanager_close(bsfc); + return err; + } + } + } + + err = mpegh_UI_SetPersistenceMemory(ctx->handle, ctx->persistency_buffer, FFMIN(persistency_size, USHRT_MAX)); + if (err != MPEGH_UI_OK && err != MPEGH_UI_OK_BUT_NO_VALID_DATA) { + av_log(bsfc, AV_LOG_ERROR, "mpegh_UI_SetPersistenceMemory() failed (%x)\n", err); + libmpeghdec_uimanager_close(bsfc); + return AVERROR_EXTERNAL; + } + + return 0; +} + +static int write_persistency_file(AVBSFContext *bsfc) +{ + MPEGHUIManagerContext *ctx = bsfc->priv_data; + FILE *persistency_file = NULL; + void *persistency_buffer = NULL; + unsigned short persistency_size = 0; + int err = 0; + + if (!ctx->persistency_file_path) { + return 0; + } + + if ((err = mpegh_UI_GetPersistenceMemory(ctx->handle, &persistency_buffer, &persistency_size)) != MPEGH_UI_OK) { + av_log(bsfc, AV_LOG_ERROR, "mpegh_UI_GetPersistenceMemory() failed (%x)\n", err); + return AVERROR_EXTERNAL; + } + + if (!strcmp(ctx->persistency_file_path, "-")) { + persistency_file = stdout; + } else { + persistency_file = avpriv_fopen_utf8(ctx->persistency_file_path, "wb"); + } + if (!persistency_file) { + err = AVERROR(errno); + av_log(bsfc, AV_LOG_ERROR, "Could not open persistency file %s: %s\n", ctx->persistency_file_path, av_err2str(err)); + return err; + } + + fwrite(persistency_buffer, 1, persistency_size, persistency_file); + err = ferror(persistency_file); + if (strcmp(ctx->persistency_file_path, "-")) { + fclose(persistency_file); + } + if (err) { + err = AVERROR(errno); + av_log(bsfc, AV_LOG_ERROR, "Could not write persistency file %s: %s\n", ctx->persistency_file_path, av_err2str(err)); + return err; + } + + return 0; +} + +static int libmpeghdec_uimanager_init(AVBSFContext *bsfc) +{ + MPEGHUIManagerContext *ctx = bsfc->priv_data; + ctx->audio_scene_xml = NULL; + + ctx->handle = mpegh_UI_Manager_Open(); + if (!ctx->handle) { + av_log(bsfc, AV_LOG_ERROR, "MPEG-H UI manager library init failed.\n"); + return AVERROR_EXTERNAL; + } + + if (ctx->audio_scene_file_path) { + if (!strcmp(ctx->audio_scene_file_path, "-")) { + ctx->audio_scene_file = stdout; + } else { + ctx->audio_scene_file = avpriv_fopen_utf8(ctx->audio_scene_file_path, "w"); + if (!ctx->audio_scene_file) { + int err = AVERROR(errno); + av_log(bsfc, AV_LOG_ERROR, "Could not open AudioScene file %s: %s\n", ctx->audio_scene_file_path, av_err2str(err)); + mpegh_UI_Manager_Close(ctx->handle); + ctx->handle = NULL; + return err; + } + } + } + + return read_persistency_file(bsfc); +} + +static int libmpeghdec_uimanager_filter(AVBSFContext *bsfc, AVPacket *pkt) +{ + MPEGHUIManagerContext *ctx = bsfc->priv_data; + int ret, err; + unsigned int output_size, flagsIn = 0, flagsOut; + unsigned long xml_length = 0, xml_buffer_size = 0; + char *xml_buffer = NULL; + + if (!ctx->handle) { + return AVERROR_UNKNOWN; + } + + ret = ff_bsf_get_packet_ref(bsfc, pkt); + if (ret < 0) + return ret; + + if ((err = mpegh_UI_FeedMHAS(ctx->handle, pkt->data, pkt->size)) != MPEGH_UI_OK) { + av_log(bsfc, AV_LOG_ERROR, "mpegh_UI_FeedMHAS() failed: %x\n", err); + av_packet_unref(pkt); + return AVERROR_EXTERNAL; + } + + if (ctx->interactivity) { + // Apply any pending ActionEvent XML and clear to mark as processed + if (ctx->action_event_xml) { + if ((err = mpegh_UI_ApplyXmlAction(ctx->handle, ctx->action_event_xml, strlen(ctx->action_event_xml), &flagsOut)) != MPEGH_UI_OK) { + av_log(bsfc, AV_LOG_ERROR, "mpegh_UI_ApplyXmlAction() failed (%x): %s\n", err, ctx->action_event_xml); + av_packet_unref(pkt); + return AVERROR_EXTERNAL; + } + av_freep(&ctx->action_event_xml); + } + + // The mpegh_UI_UpdateMHAS() function updates the existing MHAS packet (as passed to mpegh_UI_FeedMHAS()) in-place! + // Also the "output" parameter mhasLength needs to be the number of bytes fed into mpegh_UI_FeedMHAS(). + output_size = pkt->size; + av_grow_packet(pkt, MAX_PACKET_SIZE - pkt->size); + err = mpegh_UI_UpdateMHAS(ctx->handle, pkt->data, pkt->size, &output_size); + if (err == MPEGH_UI_NOT_ALLOWED) { + av_log(bsfc, AV_LOG_WARNING, "mpegh_UI_UpdateMHAS() reported MPEGH_UI_NOT_ALLOWED, input may already have UI packets. Disabling UI manager interactivity!"); + ctx->interactivity = false; + } else if (err != MPEGH_UI_OK) { + av_log(bsfc, AV_LOG_ERROR, "mpegh_UI_UpdateMHAS() failed: %x\n", err); + av_packet_unref(pkt); + return AVERROR_EXTERNAL; + } + av_shrink_packet(pkt, output_size); + } + + // Always force update the audio scene, since we otherwise miss some changes (e.g. switch groups) + flagsIn = MPEGH_UI_FORCE_UPDATE; + flagsOut = MPEGH_UI_INCOMPLETE_XML; + while (flagsOut & MPEGH_UI_INCOMPLETE_XML) { + xml_buffer = av_realloc(xml_buffer, xml_buffer_size + XML_CHUNK_SIZE); + xml_buffer_size += XML_CHUNK_SIZE; + + if ((err = mpegh_UI_GetXmlSceneState(ctx->handle, xml_buffer + xml_length, xml_buffer_size - xml_length, flagsIn, &flagsOut)) != MPEGH_UI_OK) { + av_log(bsfc, AV_LOG_ERROR, "mpegh_UI_GetXmlSceneState() failed: %x\n", err); + av_packet_unref(pkt); + av_free(xml_buffer); + return AVERROR_EXTERNAL; + } + xml_length = strlen(xml_buffer); + } + + if (ctx->audio_scene_xml == NULL || (flagsOut != MPEGH_UI_NO_CHANGE && strcmp(xml_buffer, ctx->audio_scene_xml))) { + av_freep(&ctx->audio_scene_xml); + ctx->audio_scene_xml = xml_buffer; + + if (ctx->audio_scene_file) { + fprintf(ctx->audio_scene_file, "%s", xml_buffer); + } + } else { + av_free(xml_buffer); + } + + return 0; +} + +static void libmpeghdec_uimanager_close(AVBSFContext *bsfc) +{ + MPEGHUIManagerContext *ctx = bsfc->priv_data; + if (ctx->handle) { + write_persistency_file(bsfc); + mpegh_UI_Manager_Close(ctx->handle); + ctx->handle = NULL; + } + av_freep(&ctx->audio_scene_xml); + av_freep(&ctx->action_event_xml); + av_freep(&ctx->audio_scene_file_path); + if (ctx->audio_scene_file && ctx->audio_scene_file != stdout) { + fclose(ctx->audio_scene_file); + ctx->audio_scene_file = NULL; + } + av_freep(&ctx->persistency_buffer); +} + +static const enum AVCodecID codec_ids[] = { + AV_CODEC_ID_MPEGH_3D_AUDIO, + AV_CODEC_ID_NONE, +}; + +#define OFFSET(x) offsetof(MPEGHUIManagerContext, x) +#define FILE_FLAGS (AV_OPT_FLAG_AUDIO_PARAM | AV_OPT_FLAG_BSF_PARAM) +#define IN_FLAGS (AV_OPT_FLAG_AUDIO_PARAM | AV_OPT_FLAG_BSF_PARAM | AV_OPT_FLAG_RUNTIME_PARAM) +#define OUT_FLAGS (IN_FLAGS | AV_OPT_FLAG_EXPORT) +static const AVOption libmpeghdec_uimanager_options[] = { + { "interactivity", "Whether UI Manager interactivity is enabled", OFFSET(interactivity), AV_OPT_TYPE_BOOL, { .i64 = true }, false, true, OUT_FLAGS }, + { "audio_scene", "The last MPEG-H 3DA AudioScene XML", OFFSET(audio_scene_xml), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, OUT_FLAGS | AV_OPT_FLAG_READONLY }, + { "audio_scene_file", "The file path to write the AudioScenes to", OFFSET(audio_scene_file_path), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FILE_FLAGS }, + { "action_event", "The MPEG-H 3DA ActionEvent XML to apply", OFFSET(action_event_xml), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, IN_FLAGS }, + { "persistency_file", "The persistent UI Manager data file to read from and write to", OFFSET(persistency_file_path), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FILE_FLAGS }, + { NULL }, +}; + +BSF_DEFINE_CLASS(libmpeghdec_uimanager); + +const FFBitStreamFilter ff_libmpeghdec_uimanager_bsf = { + .p.name = "libmpeghdec_uimanager", + .p.codec_ids = codec_ids, + .p.priv_class = &libmpeghdec_uimanager_class, + .priv_data_size = sizeof(MPEGHUIManagerContext), + .init = libmpeghdec_uimanager_init, + .filter = libmpeghdec_uimanager_filter, + .close = libmpeghdec_uimanager_close, +}; -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
