PR #22818 opened by Marvin Scholz (ePirat) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/22818 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/22818.patch
This PR adds basic IAMF rendering using the libspatialaudio renderer library. It allows to render multiple audio elements to various output formats, including binaural rendering. A new filter, `libspatialaudio_iamf` is added, which handles the rendering using libspatialaudio, It expects each audio element as its own stream, because AVFilter has no concept of stream groups. Additionally to make it easier to use, as using that filter correctly is quite involved and requires a lot of knowledge of IAMF itself, two new FFmpeg CLI options are added: `iamf_rendering_layout` and `iamf_rendering_iamf`. These are per-streamgroup options which control rendering to IAMF, the latter is only needed when `iamf_rendering_layout` is set to `binaural`. These options are to be used with the mix presentation stream group that should be rendered, and FFmpeg CLI will automatically construct the necessary filtergraph(s) to render the respective audio elements. An example usage of these high-level options would be: ``` ffmpeg -iamf_rendering_layout:g:2 stereo -i input.mp4 -map '[0:g:2]' output.wav ``` This would render the third stream group (as they are 0-indexed) to stereo output. (The stream group has to be an mix element, setting this option on other stream groups is ignored.) >From 73e3f99a3fe242ad851e0ae9fae4533421553750 Mon Sep 17 00:00:00 2001 From: Marvin Scholz <[email protected]> Date: Tue, 15 Oct 2024 18:44:27 +0200 Subject: [PATCH 1/3] lavf: add libspatialaudio_iamf filter Add an libspatialaudio_iamf filter which can be used for basic IAMF rendering using libspatialaudio. --- configure | 4 + doc/filters.texi | 38 +++ libavfilter/Makefile | 1 + libavfilter/af_libspatialaudio_iamf.c | 269 ++++++++++++++++ libavfilter/af_libspatialaudio_renderer.cpp | 333 ++++++++++++++++++++ libavfilter/af_libspatialaudio_renderer.h | 80 +++++ libavfilter/allfilters.c | 1 + 7 files changed, 726 insertions(+) create mode 100644 libavfilter/af_libspatialaudio_iamf.c create mode 100644 libavfilter/af_libspatialaudio_renderer.cpp create mode 100644 libavfilter/af_libspatialaudio_renderer.h diff --git a/configure b/configure index 8f9fb04115..dd8f99e985 100755 --- a/configure +++ b/configure @@ -278,6 +278,7 @@ External library support: --enable-libsmbclient enable Samba protocol via libsmbclient [no] --enable-libsnappy enable Snappy compression, needed for hap encoding [no] --enable-libsoxr enable Include libsoxr resampling [no] + --enable-libspatialaudio enable ambisonics/binaural renderer support via libspatialaudio [no] --enable-libspeex enable Speex de/encoding via libspeex [no] --enable-libsrt enable Haivision SRT protocol via libsrt [no] --enable-libssh enable SFTP protocol via libssh [no] @@ -2088,6 +2089,7 @@ EXTERNAL_LIBRARY_LIST=" libsnappy libsoxr libspeex + libspatialaudio libsrt libssh libsvtav1 @@ -4218,6 +4220,7 @@ signature_filter_deps="gpl avcodec avformat" smartblur_filter_deps="gpl swscale" sobel_opencl_filter_deps="opencl" sofalizer_filter_deps="libmysofa" +libspatialaudio_iamf_filter_deps="libspatialaudio" spp_filter_deps="gpl avcodec" spp_filter_select="idctdsp fdctdsp pixblockdsp" sr_filter_deps="avformat swscale" @@ -7384,6 +7387,7 @@ enabled libsnappy && require libsnappy snappy-c.h snappy_compress -lsnap enabled libsoxr && require libsoxr soxr.h soxr_create -lsoxr enabled libssh && require_pkg_config libssh "libssh >= 0.6.0" libssh/sftp.h sftp_init enabled libspeex && require_pkg_config libspeex speex speex/speex.h speex_decoder_init +enabled libspatialaudio && require_pkg_config_cxx libspatialaudio "spatialaudio >= 0.4.0" spatialaudio/Ambisonics.h "spaudio::AmbisonicDecoder" enabled libsrt && require_pkg_config libsrt "srt >= 1.3.0" srt/srt.h srt_socket enabled libsvtav1 && require_pkg_config libsvtav1 "SvtAv1Enc >= 0.9.0" EbSvtAv1Enc.h svt_av1_enc_init_handle enabled libsvtjpegxs && require_pkg_config libsvtjpegxs "SvtJpegxs >= 0.10.0" SvtJpegxsEnc.h svt_jpeg_xs_encoder_init diff --git a/doc/filters.texi b/doc/filters.texi index 973a93345d..0ec8da81f1 100644 --- a/doc/filters.texi +++ b/doc/filters.texi @@ -5852,6 +5852,44 @@ Modify the @var{N}-th control value. If the specified value is not valid, it is ignored and prior one is kept. @end table +@anchor{libspatialaudio_iamf} +@section libspatialaudio_iamf + +This filter provides basic IAMF (Immersive Audio) Mix Presentation rendering. + +It requires the libspatialaudio (https://github.com/videolan/libspatialaudio) +library. + +The filter expects each audio element of the mix presentation to be given as +one input stream. This can be achieved by using the @code{"join"} filter for +example. + +The filter has following options: + +@table @option +@item inputs +The number of input streams (corresponding to the audio elements). +Default value: @code{"1"} + +@item channel_layout +The output channel layout that the elements should be rendered to. +Default value: @code{"stereo"} + +@item hrtf_path +The path to a SOFA HRTF file. (Mandatory when @code{"channel_layout"} is +@code{"binaural"}) + +@end table + +@subsection Examples + +Render the first stream group, which must be an IAMF Audio element that is Scene-based +containing 3rd order Ambisonics, to Stereo: +@example +ffmpeg -i input.iamf -filter_complex "[0:g:0]join=inputs=16:channel_layout=ambisonic 3,\ +libspatialaudio_iamf=inputs=1:channel_layout=stereo[ae0]" -map '[ae0]' out.wav +@end example + @section loudnorm EBU R128 loudness normalization. Includes both dynamic and linear normalization modes. diff --git a/libavfilter/Makefile b/libavfilter/Makefile index a530cfae29..d9a1b8782f 100644 --- a/libavfilter/Makefile +++ b/libavfilter/Makefile @@ -154,6 +154,7 @@ OBJS-$(CONFIG_LADSPA_FILTER) += af_ladspa.o OBJS-$(CONFIG_LOUDNORM_FILTER) += af_loudnorm.o ebur128.o OBJS-$(CONFIG_LOWPASS_FILTER) += af_biquads.o OBJS-$(CONFIG_LOWSHELF_FILTER) += af_biquads.o +OBJS-$(CONFIG_LIBSPATIALAUDIO_IAMF_FILTER) += af_libspatialaudio_iamf.o af_libspatialaudio_renderer.o OBJS-$(CONFIG_LV2_FILTER) += af_lv2.o OBJS-$(CONFIG_MCOMPAND_FILTER) += af_mcompand.o OBJS-$(CONFIG_PAN_FILTER) += af_pan.o diff --git a/libavfilter/af_libspatialaudio_iamf.c b/libavfilter/af_libspatialaudio_iamf.c new file mode 100644 index 0000000000..e29b81b6af --- /dev/null +++ b/libavfilter/af_libspatialaudio_iamf.c @@ -0,0 +1,269 @@ +/* + * Copyright (C) 2026 Marvin Scholz + * + * Authors: Marvin Scholz <[email protected]> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA. + */ + +#include "libavutil/avstring.h" +#include "libavutil/channel_layout.h" +#include "libavutil/mem.h" +#include "libavutil/opt.h" +#include "avfilter.h" +#include "filters.h" +#include "formats.h" +#include "audio.h" + +#include "af_libspatialaudio_renderer.h" + +#define MAX_SAMPLE_COUNT 512 + +struct SpatialaudioIamfContextC { + const AVClass *class; + SpatialaudioRenderer *spr; + + // Number of inputs + unsigned nb_inputs; + + // Output channel layout + AVChannelLayout output_layout; + + // Path of the HRTF file (only used for binaural rendering) + char *hrtf_path; + + // Input frames for each stream to process + AVFrame **input_frames; + + int eof; + int64_t eof_pts; +}; + +static int filter_frames(AVFilterContext *ctx) +{ + int ret; + AVFilterLink *outlink = ctx->outputs[0]; + struct SpatialaudioIamfContextC *s = ctx->priv; + + AVFrame *out = ff_get_audio_buffer(outlink, s->input_frames[0]->nb_samples); + if (!out) + return AVERROR(ENOMEM); + + // TODO: Be more clever about how props are copied + ret = av_frame_copy_props(out, s->input_frames[0]); + if (ret < 0) + goto error; + + for (unsigned i = 0; i < ctx->nb_inputs; ++i) { + AVFrame *in = s->input_frames[i]; + + if (av_channel_layout_ambisonic_order(&in->ch_layout) < 0) + spatialaudio_renderer_add_direct_speakers(s->spr, i, + (float **)in->extended_data, in->nb_samples); + else + spatialaudio_renderer_add_hoa(s->spr, i, + (float **)in->extended_data, in->nb_samples); + } + + spatialaudio_renderer_render(s->spr, + (float **)out->extended_data, out->nb_samples); + ret = ff_filter_frame(outlink, out); + for (unsigned i = 0; i < ctx->nb_inputs; ++i) + av_frame_free(&s->input_frames[i]); + return ret; + +error: + av_frame_free(&out); + return ret; +} + +static int activate(AVFilterContext *ctx) +{ + struct SpatialaudioIamfContextC *s = ctx->priv; + + FF_FILTER_FORWARD_STATUS_BACK_ALL(ctx->outputs[0], ctx); + + for (unsigned i = 0; i < ctx->nb_inputs; i++) { + if (s->input_frames[i]) + continue; + int ret = ff_inlink_consume_samples(ctx->inputs[i], MAX_SAMPLE_COUNT, + MAX_SAMPLE_COUNT, &s->input_frames[i]); + if (ret < 0) + return ret; + + int status; + int64_t pts; + if (ff_inlink_acknowledge_status(ctx->inputs[i], &status, &pts)) { + ff_outlink_set_status(ctx->outputs[0], AVERROR_EOF, pts); + return 0; + } + + if (!s->eof && !s->input_frames[i] && ff_outlink_frame_wanted(ctx->outputs[0])) { + ff_inlink_request_frame(ctx->inputs[i]); + return 0; + } + } + + return filter_frames(ctx); +} + +static av_cold void uninit(AVFilterContext *ctx) +{ + struct SpatialaudioIamfContextC *s = ctx->priv; + spatialaudio_renderer_destroy(&s->spr); + + for (unsigned i = 0; i < s->nb_inputs && s->input_frames; ++i) { + av_frame_free(&s->input_frames[i]); + } +} + +static int get_inlink_index(AVFilterContext *ctx, AVFilterLink *inlink) +{ + for (unsigned i = 0; i < ctx->nb_inputs; ++i) { + if (ctx->inputs[i] == inlink) + return i; + } + return AVERROR_BUG; +} + +static int config_input(AVFilterLink *inlink) +{ + AVFilterContext *ctx = inlink->dst; + struct SpatialaudioIamfContextC *s = ctx->priv; + + int idx = get_inlink_index(ctx, inlink); + if (idx < 0) + return idx; + + if (av_channel_layout_ambisonic_order(&inlink->ch_layout) < 0) + return spatialaudio_renderer_config_add_direct_speakers(s->spr, &inlink->ch_layout, idx); + else + return spatialaudio_renderer_config_add_hoa(s->spr, &inlink->ch_layout, idx); +} + +static av_cold int init(AVFilterContext *ctx) +{ + struct SpatialaudioIamfContextC *s = ctx->priv; + s->spr = spatialaudio_renderer_create(ctx, s->nb_inputs); + s->input_frames = av_calloc(s->nb_inputs, sizeof(*s->input_frames)); + if (!s->spr || !s->input_frames) + return AVERROR(ENOMEM); + + for (unsigned i = 0; i < s->nb_inputs; i++) { + AVFilterPad pad = { + .type = AVMEDIA_TYPE_AUDIO, + .config_props = config_input, + }; + pad.name = av_asprintf("input_%d", i); + if (!pad.name) + return AVERROR(ENOMEM); + + int ret; + if ((ret = ff_append_inpad_free_name(ctx, &pad)) < 0) + return ret; + } + + return 0; +} + +static int config_output(AVFilterLink *outlink) +{ + int ret; + AVFilterContext *ctx = outlink->src; + struct SpatialaudioIamfContextC *s = ctx->priv; + + ret = spatialaudio_renderer_configure(s->spr, + outlink->sample_rate, MAX_SAMPLE_COUNT, s->hrtf_path, + true, &outlink->ch_layout); + if (ret < 0) + return ret; + + return 0; +} + +static int query_formats(const AVFilterContext *ctx, + AVFilterFormatsConfig **cfg_in, + AVFilterFormatsConfig **cfg_out) +{ + int ret; + struct SpatialaudioIamfContextC *s = ctx->priv; + AVFilterChannelLayouts *channel_layouts = NULL; + + static const enum AVSampleFormat sample_fmts[] = { + AV_SAMPLE_FMT_FLTP, + AV_SAMPLE_FMT_NONE + }; + + ret = ff_set_sample_formats_from_list2(ctx, cfg_in, cfg_out, sample_fmts); + if (ret < 0) + return ret; + + for (unsigned i = 0; i < ctx->nb_inputs; i++) { + const AVFilterLink *const link = ctx->inputs[i]; + if (!cfg_in[i]->channel_layouts && link->type == AVMEDIA_TYPE_AUDIO) { + ret = ff_channel_layouts_ref(ff_all_channel_layouts(), &cfg_in[i]->channel_layouts); + if (ret < 0) + return ret; + } + } + + ret = ff_add_channel_layout(&channel_layouts, &s->output_layout); + if (ret < 0) + return ret; + + ret = ff_channel_layouts_ref(channel_layouts, &cfg_out[0]->channel_layouts); + if (ret < 0) + return ret; + + return 0; +} + +#define OFFSET(x) offsetof(struct SpatialaudioIamfContextC, x) +#define FLAGS (AV_OPT_FLAG_AUDIO_PARAM | AV_OPT_FLAG_FILTERING_PARAM) + +static const AVOption libspatialaudio_iamf_options[] = { + { "inputs", "Number of input streams (audio elements)", OFFSET(nb_inputs), + AV_OPT_TYPE_UINT, { .i64 = 1 }, 0, INT_MAX, .flags = FLAGS }, + + { "channel_layout", "Output channel layout", OFFSET(output_layout), + AV_OPT_TYPE_CHLAYOUT, { .str = "stereo" }, .flags = FLAGS }, + + { "hrtf_path", "Path to the HRTF file to use for binauralization", OFFSET(hrtf_path), + AV_OPT_TYPE_STRING, { .str = NULL }, .flags = FLAGS }, + + { NULL } +}; + +AVFILTER_DEFINE_CLASS(libspatialaudio_iamf); + +static const AVFilterPad outputs[] = { + { + .name = "default", + .type = AVMEDIA_TYPE_AUDIO, + .config_props = config_output, + }, +}; + +const FFFilter ff_af_libspatialaudio_iamf = { + .p.name = "libspatialaudio_iamf", + .p.description = NULL_IF_CONFIG_SMALL("IAMF audio rendering using libspatialaudio"), + .p.priv_class = &libspatialaudio_iamf_class, + .priv_size = sizeof(struct SpatialaudioIamfContextC), + .init = init, + .uninit = uninit, + .activate = activate, + FILTER_OUTPUTS(outputs), + FILTER_QUERY_FUNC2(query_formats), +}; diff --git a/libavfilter/af_libspatialaudio_renderer.cpp b/libavfilter/af_libspatialaudio_renderer.cpp new file mode 100644 index 0000000000..52ac91b9cd --- /dev/null +++ b/libavfilter/af_libspatialaudio_renderer.cpp @@ -0,0 +1,333 @@ +/* + * Copyright (C) 2026 Marvin Scholz + * + * Authors: Marvin Scholz <[email protected]> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA. + */ + +/* Internal bridging header for libspatialaudio renderer interactions + * + * This simplifies interacting with the library from C code and is necessary + * given that we can not directly use C++ for the filter implementation as some + * FFmpeg helpers do not work in C++ code. + */ + +extern "C" { + #include "libavutil/avassert.h" + #include "libavutil/channel_layout.h" + #include "libavutil/error.h" + #include "libavutil/mem.h" +} + +#include <new> +#include <spatialaudio/Renderer.h> +#include <spatialaudio/AmbisonicCommons.h> +#include <spatialaudio/RendererMetadata.h> +#include <spatialaudio/LoudspeakerLayouts.h> + +#include "af_libspatialaudio_renderer.h" + +// Output layout map +static const std::map<spaudio::OutputLayout, AVChannelLayout> outputLayoutMap { + { spaudio::OutputLayout::Stereo, AV_CHANNEL_LAYOUT_STEREO }, + { spaudio::OutputLayout::Quad, AV_CHANNEL_LAYOUT_QUAD }, + { spaudio::OutputLayout::FivePointOne, AV_CHANNEL_LAYOUT_5POINT1_BACK }, + { spaudio::OutputLayout::FivePointOnePointTwo, AV_CHANNEL_LAYOUT_5POINT1POINT2_BACK }, + { spaudio::OutputLayout::FivePointOnePointFour, AV_CHANNEL_LAYOUT_5POINT1POINT4_BACK }, + { spaudio::OutputLayout::SevenPointOne, AV_CHANNEL_LAYOUT_7POINT1 }, + { spaudio::OutputLayout::SevenPointOnePointFour, AV_CHANNEL_LAYOUT_7POINT1POINT4_BACK }, + { spaudio::OutputLayout::ThreePointOnePointTwo, AV_CHANNEL_LAYOUT_3POINT1POINT2 }, + { spaudio::OutputLayout::Binaural, AV_CHANNEL_LAYOUT_BINAURAL }, + { spaudio::OutputLayout::TwentyTwoPointTwo, AV_CHANNEL_LAYOUT_22POINT2 }, +}; + +static spaudio::Optional<spaudio::OutputLayout> +output_layout_for_avchannellayout(const AVChannelLayout *layout) +{ + for (auto const& item : outputLayoutMap) { + if (av_channel_layout_compare(layout, &item.second) == 0) + return item.first; + } + + return spaudio::Optional<spaudio::OutputLayout>(); +} + +// Speaker names according to Rec. ITU-R BS.2094-2 +static const std::map<enum AVChannel, std::string> speakerFormatNames = { + { AV_CHAN_FRONT_LEFT, "M+030" }, // FrontLeft + { AV_CHAN_FRONT_RIGHT, "M-030" }, // FrontRight + { AV_CHAN_FRONT_CENTER, "M+000" }, // FrontCentre + { AV_CHAN_LOW_FREQUENCY, "LFE" }, // LowFrequencyEffects + { AV_CHAN_SIDE_SURROUND_LEFT, "M+110" }, // SurroundLeft + { AV_CHAN_SIDE_SURROUND_RIGHT, "M-110" }, // SurroundRight + { AV_CHAN_FRONT_LEFT_OF_CENTER, "M+022" }, // FrontLeftOfCentre + { AV_CHAN_FRONT_RIGHT_OF_CENTER, "M-022" }, // FrontRightOfCentre + { AV_CHAN_BACK_CENTER, "M+180" }, // BackCentre + { AV_CHAN_SIDE_LEFT, "M+090" }, // SideLeft + { AV_CHAN_SIDE_RIGHT, "M-090" }, // SideRight + { AV_CHAN_TOP_CENTER, "T+000" }, // TopCentre + { AV_CHAN_TOP_FRONT_LEFT, "U+045" }, // TopFrontLeftMid + { AV_CHAN_TOP_FRONT_CENTER, "U+000" }, // TopFrontCentre + { AV_CHAN_TOP_FRONT_RIGHT, "U-045" }, // TopFrontRightMid + { AV_CHAN_TOP_SURROUND_LEFT, "U+110" }, // TopSurroundLeft + { AV_CHAN_TOP_BACK_CENTER, "U+180" }, // TopBackCentre + { AV_CHAN_TOP_SURROUND_RIGHT, "U-110" }, // TopSurroundRight + { AV_CHAN_TOP_SIDE_LEFT, "U+090" }, // TopSideLeft + { AV_CHAN_TOP_SIDE_RIGHT, "U-090" }, // TopSideRight + { AV_CHAN_BOTTOM_FRONT_CENTER, "B+000" }, // BottomFrontCentre + { AV_CHAN_BOTTOM_FRONT_LEFT, "B+045" }, // BottomFrontLeftMid + { AV_CHAN_BOTTOM_FRONT_RIGHT, "B-045" }, // BottomFrontRightMid + { AV_CHAN_WIDE_LEFT, "M+060" }, // FrontLeftWide + { AV_CHAN_WIDE_RIGHT, "M-060" }, // FrontRightWide + // N/A + // N/A + { AV_CHAN_BACK_LEFT, "M+135" }, // BackLeftMid + { AV_CHAN_BACK_RIGHT, "M-135" }, // BackRightMid + { AV_CHAN_TOP_BACK_LEFT, "U+135" }, // TopBackLeftMid + { AV_CHAN_TOP_BACK_RIGHT, "U-135" }, // TopBackRightMid +}; + +static bool get_format_name_for_avchannel(enum AVChannel channel, + std::string &name) +{ + auto it = speakerFormatNames.find(channel); + if (it == speakerFormatNames.end()) + return false; + + name = it->second; + return true; +} + +// Channel layouts to audio pack ID strings mapping +static const std::map<std::string, AVChannelLayout> formatIdToChanLayout { + { "AP_00010001", AV_CHANNEL_LAYOUT_MONO }, + { "AP_00010002", AV_CHANNEL_LAYOUT_STEREO }, + { "AP_00010003", AV_CHANNEL_LAYOUT_5POINT1 }, // Also AP_0001000c + { "AP_00010004", AV_CHANNEL_LAYOUT_5POINT1POINT2_BACK }, + { "AP_00010005", AV_CHANNEL_LAYOUT_5POINT1POINT4_BACK }, + // AP_00010010 (4+5+1) + // AP_00010007 (3+7+0) + // AP_00010008 (4+9+0) + // AP_00010009 (9+10+3) + { "AP_0001000f", AV_CHANNEL_LAYOUT_7POINT1 }, + { "AP_00010017", AV_CHANNEL_LAYOUT_7POINT1POINT4_BACK }, +}; + +static std::string +audio_pack_id_for_avchannellayout(const AVChannelLayout *layout) +{ + for (auto const& item : formatIdToChanLayout) { + int64_t mask = item.second.u.mask; + if (layout->nb_channels != item.second.nb_channels) + continue; + if (av_channel_layout_subset(layout, mask) == mask) + return item.first; + } + return std::string(); +} + +struct StreamMeta +{ + spaudio::TypeDefinition type; + union { + spaudio::HoaMetadata *hoa; + std::vector<spaudio::DirectSpeakerMetadata> *ds; + }; +}; + +struct SpatialaudioRenderer +{ + void *logctx; + spaudio::Renderer r; + unsigned currentTrackInd; + std::vector<StreamMeta> streams_meta; + spaudio::StreamInformation streams_info; +}; + +void spatialaudio_renderer_destroy(SpatialaudioRenderer **renderer) +{ + if (*renderer) + ((*renderer)->r).~Renderer(); + av_freep(renderer); +} + +SpatialaudioRenderer *spatialaudio_renderer_create(void *logctx, size_t nb_streams) +{ + SpatialaudioRenderer *renderer = + static_cast<SpatialaudioRenderer *>(av_mallocz(sizeof(SpatialaudioRenderer))); + if (!renderer) + return NULL; + try { + new(&renderer->r) spaudio::Renderer(); + new(&renderer->streams_meta) std::vector<StreamMeta>(nb_streams); + new(&renderer->streams_info) spaudio::StreamInformation(); + } catch(const std::bad_alloc &e) { + spatialaudio_renderer_destroy(&renderer); + } + renderer->logctx = logctx; + renderer->currentTrackInd = 0; + return renderer; +} + +int spatialaudio_renderer_config_add_hoa(SpatialaudioRenderer *renderer, + const AVChannelLayout *layout, size_t idx) +{ + av_assert0(idx < renderer->streams_meta.size()); + try { + spaudio::HoaMetadata *hoa = new spaudio::HoaMetadata; + hoa->normalization = "SN3D"; + for (size_t i = 0; i < layout->nb_channels; ++i) { + int order, degree; + spaudio::ComponentToOrderAndDegree(i, true, order, degree); + hoa->trackInds.push_back(renderer->currentTrackInd); + hoa->orders.push_back(order); + hoa->degrees.push_back(degree); + + renderer->streams_info.typeDefinition.push_back(spaudio::TypeDefinition::HOA); + renderer->streams_info.nChannels++; + renderer->currentTrackInd++; + } + StreamMeta meta = { + spaudio::TypeDefinition::HOA, + { .hoa = hoa }, + }; + renderer->streams_meta.at(idx) = meta; + return 0; + } catch (const std::bad_array_new_length &e) { + return AVERROR(ENOMEM); + } catch (const std::bad_alloc &e) { + return AVERROR(ERANGE); + } +} + +int spatialaudio_renderer_config_add_direct_speakers(SpatialaudioRenderer *renderer, + const AVChannelLayout *layout, size_t idx) +{ + av_assert0(idx < renderer->streams_meta.size()); + char layout_name[128]; + int ret = av_channel_layout_describe(layout, layout_name, sizeof(layout_name)); + if (ret < 0) + return ret; + + try { + std::vector<spaudio::DirectSpeakerMetadata> *ds = + new std::vector<spaudio::DirectSpeakerMetadata>(layout->nb_channels); + + std::string formatID = audio_pack_id_for_avchannellayout(layout); + if (formatID.empty()) { + av_log(renderer->logctx, AV_LOG_ERROR, + "Unsupported channel layout '%s' encountered\n", layout_name); + return AVERROR(ENOTSUP); + } + av_log(renderer->logctx, AV_LOG_VERBOSE, + "Mapping channel layout '%s' -> %s\n", layout_name, formatID.c_str()); + + for (int i = 0; i < layout->nb_channels; ++i) { + char name[32]; + enum AVChannel channel = av_channel_layout_channel_from_index(layout, i); + av_channel_name(name, sizeof(name), channel); + + if (!get_format_name_for_avchannel(channel, ds->at(i).speakerLabel)) { + av_log(renderer->logctx, AV_LOG_ERROR, + "Unable to get spaudio channel label for %s\n", name); + delete ds; + return AVERROR(ENOTSUP); + } + av_log(renderer->logctx, AV_LOG_VERBOSE, + "Mapping channel %i: %s -> %s\n", i, name, ds->at(i).speakerLabel.c_str()); + + ds->at(i).trackInd = renderer->currentTrackInd; + ds->at(i).audioPackFormatID = formatID; + + renderer->streams_info.typeDefinition.push_back(spaudio::TypeDefinition::DirectSpeakers); + renderer->streams_info.nChannels++; + renderer->currentTrackInd++; + } + StreamMeta meta = { + spaudio::TypeDefinition::DirectSpeakers, + { .ds = ds }, + }; + renderer->streams_meta.at(idx) = meta; + return 0; + } catch (const std::bad_array_new_length &e) { + return AVERROR(ENOMEM); + } catch (const std::bad_alloc &e) { + return AVERROR(ERANGE); + } +} + +int spatialaudio_renderer_configure(SpatialaudioRenderer *renderer, + unsigned samplerate, size_t num_samples, const char *hrtf_path, + bool use_lfe_binaural, const AVChannelLayout *output_layout) +{ + spaudio::Optional<spaudio::OutputLayout> layout = + output_layout_for_avchannellayout(output_layout); + + if (!layout.hasValue()) { + char name[128]; + int ret = av_channel_layout_describe(output_layout, name, sizeof(name)); + if (ret < 0) + return ret; + + av_log(renderer->logctx, AV_LOG_ERROR, "Unsupported rendering output layout %s\n", name); + return AVERROR(ENOTSUP); + } + if (layout.value() == spaudio::OutputLayout::Binaural && + (!hrtf_path || !hrtf_path[0])) { + av_log(renderer->logctx, AV_LOG_ERROR, + "Binaural rendering requires HRTF file path to be specified\n"); + return AVERROR(EINVAL); + } + + try { + return renderer->r.Configure(layout.value(), 3, samplerate, num_samples, + renderer->streams_info, (hrtf_path) ? hrtf_path : "", use_lfe_binaural); + } catch (const std::bad_array_new_length &e) { + return AVERROR(ENOMEM); + } catch (const std::bad_alloc &e) { + return AVERROR(ERANGE); + } +} + +void spatialaudio_renderer_add_hoa(SpatialaudioRenderer *renderer, + size_t idx, float **samples, size_t num_samples) +{ + av_assert0(idx < renderer->streams_meta.size()); + + StreamMeta meta = renderer->streams_meta.at(idx); + av_assert0(meta.type == spaudio::TypeDefinition::HOA); + + renderer->r.AddHoa(samples, num_samples, *meta.hoa); +} + +void spatialaudio_renderer_add_direct_speakers(SpatialaudioRenderer *renderer, + size_t idx, float **samples, size_t num_samples) +{ + av_assert0(idx < renderer->streams_meta.size()); + + StreamMeta meta = renderer->streams_meta.at(idx); + av_assert0(meta.type == spaudio::TypeDefinition::DirectSpeakers); + + for (size_t i = 0; i < meta.ds->size(); ++i) { + renderer->r.AddDirectSpeaker(samples[i], num_samples, meta.ds->at(i)); + } +} + +void spatialaudio_renderer_render(SpatialaudioRenderer *renderer, + float **output, size_t num_samples) +{ + renderer->r.GetRenderedAudio(output, num_samples); +} diff --git a/libavfilter/af_libspatialaudio_renderer.h b/libavfilter/af_libspatialaudio_renderer.h new file mode 100644 index 0000000000..217860931d --- /dev/null +++ b/libavfilter/af_libspatialaudio_renderer.h @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2026 Marvin Scholz + * + * Authors: Marvin Scholz <[email protected]> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA. + */ + +#ifndef AVFILTER_LIBSPATIALAUDIO_RENDERER_H +#define AVFILTER_LIBSPATIALAUDIO_RENDERER_H + +#include <stddef.h> +#include <stdbool.h> + +#include "libavutil/channel_layout.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SpatialaudioRenderer SpatialaudioRenderer; + +/** + * Create a new renderer context + * + * Allocates and initializes a new renderer context for the + * given number of streams (audio elements). If the context + * can't be allocated, NULL is returned. + */ +SpatialaudioRenderer *spatialaudio_renderer_create(void *logctx, size_t nb_streams); + +void spatialaudio_renderer_destroy(SpatialaudioRenderer **renderer); + +/** + * Add an HOA stream to renderer config + */ +int spatialaudio_renderer_config_add_hoa(SpatialaudioRenderer *renderer, + const AVChannelLayout *layout, size_t idx); + +/** + * Add a direct speakers (channel-based) stream to renderer config + */ +int spatialaudio_renderer_config_add_direct_speakers(SpatialaudioRenderer *renderer, + const AVChannelLayout *layout, size_t idx); + +/** + * Configure the renderer + * + * Finalize the renderer configuration, all streams need to be + * added to the configuration before this is called. + */ +int spatialaudio_renderer_configure(SpatialaudioRenderer *renderer, + unsigned samplerate, size_t num_samples, const char *hrtf_path, + bool use_lfe_binaural, const AVChannelLayout *output_layout); + +void spatialaudio_renderer_add_hoa(SpatialaudioRenderer *renderer, + size_t idx, float **samples, size_t num_samples); + +void spatialaudio_renderer_add_direct_speakers(SpatialaudioRenderer *renderer, + size_t idx, float **samples, size_t num_samples); + +void spatialaudio_renderer_render(SpatialaudioRenderer *renderer, + float **output, size_t num_samples); + +#ifdef __cplusplus +} +#endif +#endif // AVFILTER_LIBSPATIALAUDIO_RENDERER_H diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c index e26859e159..168d921f64 100644 --- a/libavfilter/allfilters.c +++ b/libavfilter/allfilters.c @@ -137,6 +137,7 @@ extern const FFFilter ff_af_highpass; extern const FFFilter ff_af_highshelf; extern const FFFilter ff_af_join; extern const FFFilter ff_af_ladspa; +extern const FFFilter ff_af_libspatialaudio_iamf; extern const FFFilter ff_af_loudnorm; extern const FFFilter ff_af_lowpass; extern const FFFilter ff_af_lowshelf; -- 2.52.0 >From 4aedc890a3ce5dd75391bc253e030ff2741a58a7 Mon Sep 17 00:00:00 2001 From: Marvin Scholz <[email protected]> Date: Wed, 8 Apr 2026 13:27:24 +0200 Subject: [PATCH 2/3] fftools: add option for IAMF rendering Adds the new stream-group option `iamf_rendering_layout` which can be used to render a specific IAMF Mix Presentation stream group to the given output channel layout. For example -iamf_rendering_layout:g:2 stereo would render the second stream group, which has to be a mix presentation one, to a stereo output layout. The rendered stream can be accessed in the -map option as `[0:g:2]`. --- fftools/ffmpeg.h | 1 + fftools/ffmpeg_demux.c | 143 ++++++++++++++++++++++++++++++++++++++++- fftools/ffmpeg_opt.c | 3 + 3 files changed, 146 insertions(+), 1 deletion(-) diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h index 3a19e5878d..1ea86e2d2d 100644 --- a/fftools/ffmpeg.h +++ b/fftools/ffmpeg.h @@ -245,6 +245,7 @@ typedef struct OptionsContext { SpecifierOptList max_muxing_queue_size; SpecifierOptList muxing_queue_data_threshold; SpecifierOptList guess_layout_max; + SpecifierOptList iamf_rendering_layout; SpecifierOptList apad; SpecifierOptList discard; SpecifierOptList disposition; diff --git a/fftools/ffmpeg_demux.c b/fftools/ffmpeg_demux.c index 04ee2cc2ed..dec0c213ee 100644 --- a/fftools/ffmpeg_demux.c +++ b/fftools/ffmpeg_demux.c @@ -22,9 +22,11 @@ #include "ffmpeg.h" #include "ffmpeg_sched.h" #include "ffmpeg_utils.h" +#include "config_components.h" #include "libavutil/avassert.h" #include "libavutil/avstring.h" +#include "libavutil/bprint.h" #include "libavutil/display.h" #include "libavutil/error.h" #include "libavutil/intreadwrite.h" @@ -35,12 +37,14 @@ #include "libavutil/pixdesc.h" #include "libavutil/time.h" #include "libavutil/timestamp.h" +#include "libavutil/iamf.h" #include "libavcodec/bsf.h" #include "libavcodec/packet.h" #include "libavformat/avformat.h" + typedef struct DemuxStream { InputStream ist; @@ -1865,6 +1869,139 @@ fail: return ret; } +#if CONFIG_LIBSPATIALAUDIO_IAMF_FILTER +static AVStreamGroup *find_stream_group_by_id(const AVFormatContext *ic, + enum AVStreamGroupParamsType type, int id) +{ + for (int i = 0; i < ic->nb_stream_groups; i++) { + if (ic->stream_groups[i]->type == type && ic->stream_groups[i]->id == id) + return ic->stream_groups[i]; + } + return NULL; +} +#endif + +static int istg_parse_iamf_mix_presentation(const OptionsContext *o, Demuxer *d, InputStreamGroup *istg) +{ + AVFormatContext *ic = d->f.ctx; + AVStreamGroup *stg = istg->stg; + const AVIAMFMixPresentation *mp = stg->params.iamf_mix_presentation; + + const char *out_ch_layout_str = NULL; + opt_match_per_stream_group_str(istg, &o->iamf_rendering_layout, ic, stg, &out_ch_layout_str); + if (!out_ch_layout_str) + return 0; + +#if !CONFIG_LIBSPATIALAUDIO_IAMF_FILTER + av_log(istg, AV_LOG_ERROR, "Rendering IAMF requires the libspatialaudio_iamf filter"); + return AVERROR(ENOTSUP); +#else + unsigned streams_nb = 0; + AVBPrint bp, iamf_bp; + + if (mp->nb_submixes > 1) { + av_log(istg, AV_LOG_ERROR, "Rendering more than one IAMF sub-mix is not yet supported"); + return AVERROR_PATCHWELCOME; + } + + av_bprint_init(&bp, 0, AV_BPRINT_SIZE_UNLIMITED); + av_bprint_init(&iamf_bp, 0, AV_BPRINT_SIZE_UNLIMITED); + + for (int i = 0; i < mp->nb_submixes; i++) { + AVIAMFSubmix *submix = mp->submixes[i]; + av_log(istg, AV_LOG_TRACE, "processing IAMF mix presentation submix %d:" + " nb_elements=%d, default_mix_gain=%f\n", i, submix->nb_elements, + av_q2d(submix->default_mix_gain)); + for (int k = 0; k < submix->nb_elements; k++) { + const AVIAMFSubmixElement *submix_element = submix->elements[k]; + av_log(istg, AV_LOG_TRACE, "IAMF submix element %d: audio_element_id=%d, default_mix_gain=%f\n", + k, submix_element->audio_element_id, av_q2d(submix_element->default_mix_gain)); + + const AVStreamGroup *audio_element_group = + find_stream_group_by_id(ic, AV_STREAM_GROUP_PARAMS_IAMF_AUDIO_ELEMENT, + submix_element->audio_element_id); + if (!audio_element_group) { + av_log(istg, AV_LOG_ERROR, "couldn't find audio element stream group id %i referenced by" + " this mix presentation.", submix_element->audio_element_id); + av_bprint_finalize(&bp, NULL); + av_bprint_finalize(&iamf_bp, NULL); + return AVERROR_INVALIDDATA; + } + + const AVIAMFAudioElement *audio_element = audio_element_group->params.iamf_audio_element; + + // Merge Audio Element streams into a single stream with the "proper" + // channel layout, so that the filter can just look at that instead of + // needing the IAMF metadata. + switch (audio_element->audio_element_type) + { + case AV_IAMF_AUDIO_ELEMENT_TYPE_CHANNEL: + case AV_IAMF_AUDIO_ELEMENT_TYPE_SCENE: { + if (audio_element->nb_layers != 1) { + if (audio_element->audio_element_type == AV_IAMF_AUDIO_ELEMENT_TYPE_CHANNEL) { + av_log(istg, AV_LOG_ERROR, "IAMF channel audio elements " + " with more than one layer are unsupported\n"); + av_bprint_finalize(&bp, NULL); + av_bprint_finalize(&iamf_bp, NULL); + return AVERROR_PATCHWELCOME; + } else { + // Scene-based elements must only have one layer + av_bprint_finalize(&bp, NULL); + av_bprint_finalize(&iamf_bp, NULL); + return AVERROR_INVALIDDATA; + } + } + + const AVIAMFLayer *layer = audio_element->layers[0]; + for (unsigned si = 0; si < audio_element_group->nb_streams; si++) { + av_bprintf(&bp, "[%d:g:%d:%d]", d->f.index, audio_element_group->index, si); + } + + av_bprintf(&bp, "join=inputs=%d:channel_layout=", audio_element_group->nb_streams); + av_channel_layout_describe_bprint(&layer->ch_layout, &bp); + av_bprintf(&bp, "[%d:g:%d];", d->f.index, audio_element_group->index); + + av_bprintf(&iamf_bp, "[%d:g:%d]", d->f.index, audio_element_group->index); + streams_nb++; + break; + } + default: + return AVERROR_INVALIDDATA; + } + } + } + + av_bprintf(&iamf_bp, "libspatialaudio_iamf=inputs=%d:channel_layout=%s[%d:g:%d]", + streams_nb, out_ch_layout_str, d->f.index, stg->index); + + char *iamf_filter_str; + int ret = av_bprint_finalize(&iamf_bp, &iamf_filter_str); + if (ret < 0) { + av_bprint_finalize(&bp, NULL); + return ret; + } + av_bprintf(&bp, "\n%s", iamf_filter_str); + av_freep(&iamf_filter_str); + + char *graph_str; + ret = av_bprint_finalize(&bp, &graph_str); + if (ret < 0) + return ret; + + av_log(istg, AV_LOG_VERBOSE, "IAMF mix presentation rendering fg:\n%s\n", + graph_str); + ret = fg_create(NULL, &graph_str, d->sch, NULL); + if (ret < 0) { + av_freep(&graph_str); + return ret; + } + + istg->fg = filtergraphs[nb_filtergraphs-1]; + istg->fg->is_internal = 1; + return 0; +#endif +} + static int istg_add(const OptionsContext *o, Demuxer *d, AVStreamGroup *stg) { DemuxStreamGroup *dsg; @@ -1883,10 +2020,14 @@ static int istg_add(const OptionsContext *o, Demuxer *d, AVStreamGroup *stg) if (ret < 0) return ret; break; + case AV_STREAM_GROUP_PARAMS_IAMF_MIX_PRESENTATION: + ret = istg_parse_iamf_mix_presentation(o, d, istg); + if (ret < 0) + return ret; + break; default: break; } - return 0; } diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c index 48e6816c19..05be5608b5 100644 --- a/fftools/ffmpeg_opt.c +++ b/fftools/ffmpeg_opt.c @@ -2082,6 +2082,9 @@ const OptionDef options[] = { { "guess_layout_max", OPT_TYPE_INT, OPT_AUDIO | OPT_PERSTREAM | OPT_EXPERT | OPT_INPUT, { .off = OFFSET(guess_layout_max) }, "set the maximum number of channels to try to guess the channel layout" }, + { "iamf_rendering_layout", OPT_TYPE_STRING, OPT_AUDIO | OPT_PERSTREAM | OPT_EXPERT | OPT_INPUT, + { .off = OFFSET(iamf_rendering_layout) }, + "set the output channel layout to render an IAMF Mix Presentation as" }, /* subtitle options */ { "sn", OPT_TYPE_BOOL, OPT_SUBTITLE | OPT_OFFSET | OPT_INPUT | OPT_OUTPUT, -- 2.52.0 >From ea46b74a6f280a3a568cd54dc5518a41ff966d47 Mon Sep 17 00:00:00 2001 From: Marvin Scholz <[email protected]> Date: Wed, 15 Apr 2026 01:23:14 +0200 Subject: [PATCH 3/3] fftools: add HRTF option for IAMF binaural rendering --- fftools/ffmpeg.h | 1 + fftools/ffmpeg_demux.c | 11 ++++++++--- fftools/ffmpeg_opt.c | 3 +++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h index 1ea86e2d2d..14001b0fb1 100644 --- a/fftools/ffmpeg.h +++ b/fftools/ffmpeg.h @@ -246,6 +246,7 @@ typedef struct OptionsContext { SpecifierOptList muxing_queue_data_threshold; SpecifierOptList guess_layout_max; SpecifierOptList iamf_rendering_layout; + SpecifierOptList iamf_rendering_hrtf; SpecifierOptList apad; SpecifierOptList discard; SpecifierOptList disposition; diff --git a/fftools/ffmpeg_demux.c b/fftools/ffmpeg_demux.c index dec0c213ee..fa2d4b98ba 100644 --- a/fftools/ffmpeg_demux.c +++ b/fftools/ffmpeg_demux.c @@ -1886,8 +1886,8 @@ static int istg_parse_iamf_mix_presentation(const OptionsContext *o, Demuxer *d, AVFormatContext *ic = d->f.ctx; AVStreamGroup *stg = istg->stg; const AVIAMFMixPresentation *mp = stg->params.iamf_mix_presentation; - const char *out_ch_layout_str = NULL; + opt_match_per_stream_group_str(istg, &o->iamf_rendering_layout, ic, stg, &out_ch_layout_str); if (!out_ch_layout_str) return 0; @@ -1971,8 +1971,13 @@ static int istg_parse_iamf_mix_presentation(const OptionsContext *o, Demuxer *d, } } - av_bprintf(&iamf_bp, "libspatialaudio_iamf=inputs=%d:channel_layout=%s[%d:g:%d]", - streams_nb, out_ch_layout_str, d->f.index, stg->index); + const char *hrtf_path = NULL; + av_bprintf(&iamf_bp, "libspatialaudio_iamf=inputs=%d:channel_layout=%s", + streams_nb, out_ch_layout_str); + opt_match_per_stream_group_str(istg, &o->iamf_rendering_hrtf, ic, stg, &hrtf_path); + if (hrtf_path) + av_bprintf(&iamf_bp, ":hrtf_path='%s'", hrtf_path); + av_bprintf(&iamf_bp, "[%d:g:%d]", d->f.index, stg->index); char *iamf_filter_str; int ret = av_bprint_finalize(&iamf_bp, &iamf_filter_str); diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c index 05be5608b5..e2ad2bfe23 100644 --- a/fftools/ffmpeg_opt.c +++ b/fftools/ffmpeg_opt.c @@ -2085,6 +2085,9 @@ const OptionDef options[] = { { "iamf_rendering_layout", OPT_TYPE_STRING, OPT_AUDIO | OPT_PERSTREAM | OPT_EXPERT | OPT_INPUT, { .off = OFFSET(iamf_rendering_layout) }, "set the output channel layout to render an IAMF Mix Presentation as" }, + { "iamf_rendering_hrtf", OPT_TYPE_STRING, OPT_AUDIO | OPT_PERSTREAM | OPT_EXPERT | OPT_INPUT, + { .off = OFFSET(iamf_rendering_hrtf) }, + "set HRTF SOFA file path (needed for binaural IAMF rendering)" }, /* subtitle options */ { "sn", OPT_TYPE_BOOL, OPT_SUBTITLE | OPT_OFFSET | OPT_INPUT | OPT_OUTPUT, -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
